sh: Change interaction of command substitution and here documents.

If a command substitution contains a newline token, this no longer starts
here documents of outer commands. This way, we follow POSIX's idea of the
command substitution being a separate script more closely. It also matches
other shells better and is consistent with newline characters in quotes not
starting here documents.

The extension tested in parser/heredoc3.0 ($(cat <<EOF)\ntext\nEOF\n)
continues to be supported.

In particular, this change allows things like
  cat <<EOF && echo `pwd`
(a `` command substitution after a here document)
which formerly silently used an empty file as the here document, because the
EOF of the inner command "pwd" also forced an empty here document.

bin/sh/parser.c

@@ -973,6 +973,8 @@ parsebackq(char *out, struct nodelist **pbqlist,
 	const int bq_startlinno = plinno;
 	char *volatile ostr = NULL;
 	struct parsefile *const savetopfile = getcurrentfile();
+	struct heredoc *const saveheredoclist = heredoclist;
+	struct heredoc *here;
 
 	str = NULL;
 	if (setjmp(jmploc.loc)) {
@@ -981,6 +983,7 @@ parsebackq(char *out, struct nodelist **pbqlist,
 			ckfree(str);
 		if (ostr)
 			ckfree(ostr);
+		heredoclist = saveheredoclist;
 		handler = savehandler;
 		if (exception == EXERROR) {
 			startlinno = bq_startlinno;
@@ -995,6 +998,7 @@ parsebackq(char *out, struct nodelist **pbqlist,
 		memcpy(str, stackblock(), savelen);
 	}
 	handler = &jmploc;
+	heredoclist = NULL;
 	INTON;
         if (oldstyle) {
                 /* We must read until the closing backquote, giving special
@@ -1091,21 +1095,26 @@ parsebackq(char *out, struct nodelist **pbqlist,
 	while (stackblocksize() <= savelen)
 		growstackblock();
 	STARTSTACKSTR(out);
+	INTOFF;
 	if (str) {
 		memcpy(out, str, savelen);
 		STADJUST(savelen, out);
-		INTOFF;
 		ckfree(str);
 		str = NULL;
-		INTON;
 	}
 	if (ostr) {
-		INTOFF;
 		ckfree(ostr);
 		ostr = NULL;
-		INTON;
+	}
+	here = saveheredoclist;
+	if (here != NULL) {
+		while (here->next != NULL)
+			here = here->next;
+		here->next = heredoclist;
+		heredoclist = saveheredoclist;
 	}
 	handler = savehandler;
+	INTON;
 	if (quoted)
 		USTPUTC(CTLBACKQ | CTLQUOTE, out);
 	else

tools/regression/bin/sh/parser/heredoc4.0

@@ -0,0 +1,44 @@
+# $FreeBSD$
+
+failures=0
+
+check() {
+	if ! eval "[ $* ]"; then
+		echo "Failed: $*"
+		: $((failures += 1))
+	fi
+}
+
+f() {
+	cat <<EOF && echo `echo bar`
+foo
+EOF
+}
+check '"`f`" = "foo
+bar"'
+
+f() {
+	cat <<EOF && echo $(echo bar)
+foo
+EOF
+}
+check '"$(f)" = "foo
+bar"'
+
+f() {
+	echo `echo bar` && cat <<EOF
+foo
+EOF
+}
+check '"`f`" = "bar
+foo"'
+
+f() {
+	echo $(echo bar) && cat <<EOF
+foo
+EOF
+}
+check '"$(f)" = "bar
+foo"'
+
+exit $((failures != 0))

tools/regression/bin/sh/parser/heredoc5.0

@@ -0,0 +1,56 @@
+# $FreeBSD$
+
+failures=0
+
+check() {
+	if ! eval "[ $* ]"; then
+		echo "Failed: $*"
+		: $((failures += 1))
+	fi
+}
+
+f() {
+	cat <<EOF && echo `cat <<EOF
+bar
+EOF
+`
+foo
+EOF
+}
+check '"`f`" = "foo
+bar"'
+
+f() {
+	cat <<EOF && echo $(cat <<EOF
+bar
+EOF
+)
+foo
+EOF
+}
+check '"$(f)" = "foo
+bar"'
+
+f() {
+	echo `cat <<EOF
+bar
+EOF
+` && cat <<EOF
+foo
+EOF
+}
+check '"`f`" = "bar
+foo"'
+
+f() {
+	echo $(cat <<EOF
+bar
+EOF
+) && cat <<EOF
+foo
+EOF
+}
+check '"$(f)" = "bar
+foo"'
+
+exit $((failures != 0))