cmd/cue/cmd: show incorrect handling of binary files when embedded

These new tests show that CUE will treat input bytes as UTF-8
when embedding a file with `type=binary`.

Since txtar can only store files with valid UTF8,
and including any non-printable characters in a txtar is error-prone,
add a new strconv-unquote command to use Go string quoting instead.

For #3741.

Signed-off-by: nichtsundniemand <[email protected]>
Change-Id: Ic2703d80b922d66d6d37b8044e2b8a986aa58b2d
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1209312
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Roger Peppe <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

Author: nichtsundniemand

cmd/cue/cmd/script_test.go

@@ -238,6 +238,18 @@ func TestScript(t *testing.T) {
 					ts.Check(err)
 				}
 			},
+			// strconv-unquote treats each argument as a go quoted string, unquotes it and prints
+			// them, separated by newlines, to stdout
+			"strconv-unquote": func(ts *testscript.TestScript, neg bool, args []string) {
+				if neg {
+					ts.Fatalf("usage: strconv-unquote args...")
+				}
+				for _, quoted := range args {
+					s, err := strconv.Unquote(quoted)
+					ts.Check(err)
+					fmt.Fprintln(ts.Stdout(), s)
+				}
+			},
 		},
 		Setup: func(e *testscript.Env) error {
 			// If a testscript loads CUE packages but forgot to set up a cue.mod,

cmd/cue/cmd/testdata/script/embed.txtar

@@ -4,6 +4,12 @@ env CUE_EXPERIMENT=embed=0
 cmp stderr out/noembed
 env CUE_EXPERIMENT=
 
+# Create test files for type=binary using strconv-unquote as txtar can only hold UTF8.
+strconv-unquote '"\xef\xbb\xbfHello, 世界"'
+cp stdout test-utf8-bom.binary
+strconv-unquote "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7"
+cp stdout test-nonutf8.binary
+
 exec cue eval
 cmp stdout out/eval
 
@@ -39,6 +45,12 @@ f: _ @embed(file="openapi.json", type=openapi)
 
 g: _ @embed(file="openapi.json") // test no auto mode!
 
+h: _ @embed(file="test-utf8-bom.binary", type=binary)
+
+i: _ @embed(file="test-utf8-bom.binary", type=text)
+
+j: _ @embed(file="test-nonutf8.binary", type=binary)
+
 special: {
 	// These are all valid.
 	underscoreFile: _ @embed(file="y/_test.json")
@@ -213,6 +225,18 @@ g: {
         }
     }
 }
+h: '''
+    Hello, 世界
+
+    '''
+i: """
+    Hello, 世界
+
+    """
+j: '''
+    ��������
+
+    '''
 special: {
     underscoreFile: {
         z: 45
@@ -273,6 +297,18 @@ g: {
 		}
 	}
 }
+h: '''
+	Hello, 世界
+
+	'''
+i: """
+	Hello, 世界
+
+	"""
+j: '''
+	��������
+
+	'''
 special: {
 	// These are all valid.
 	underscoreFile: z:  45