1// Copyright 2018 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package genrule
16
17import (
18	"os"
19	"regexp"
20	"testing"
21
22	"android/soong/android"
23
24	"github.com/google/blueprint/proptools"
25)
26
27func TestMain(m *testing.M) {
28	os.Exit(m.Run())
29}
30
31var prepareForGenRuleTest = android.GroupFixturePreparers(
32	android.PrepareForTestWithArchMutator,
33	android.PrepareForTestWithDefaults,
34
35	android.PrepareForTestWithFilegroup,
36	PrepareForTestWithGenRuleBuildComponents,
37	android.FixtureRegisterWithContext(func(ctx android.RegistrationContext) {
38		ctx.RegisterModuleType("tool", toolFactory)
39		ctx.RegisterModuleType("output", outputProducerFactory)
40	}),
41	android.FixtureMergeMockFs(android.MockFS{
42		"tool":       nil,
43		"tool_file1": nil,
44		"tool_file2": nil,
45		"in1":        nil,
46		"in2":        nil,
47		"in1.txt":    nil,
48		"in2.txt":    nil,
49		"in3.txt":    nil,
50	}),
51)
52
53func testGenruleBp() string {
54	return `
55		tool {
56			name: "tool",
57		}
58
59		filegroup {
60			name: "tool_files",
61			srcs: [
62				"tool_file1",
63				"tool_file2",
64			],
65		}
66
67		filegroup {
68			name: "1tool_file",
69			srcs: [
70				"tool_file1",
71			],
72		}
73
74		filegroup {
75			name: "ins",
76			srcs: [
77				"in1",
78				"in2",
79			],
80		}
81
82		filegroup {
83			name: "1in",
84			srcs: [
85				"in1",
86			],
87		}
88
89		filegroup {
90			name: "empty",
91		}
92	`
93}
94
95func TestGenruleCmd(t *testing.T) {
96	testcases := []struct {
97		name string
98		prop string
99
100		allowMissingDependencies bool
101
102		err    string
103		expect string
104	}{
105		{
106			name: "empty location tool",
107			prop: `
108				tools: ["tool"],
109				out: ["out"],
110				cmd: "$(location) > $(out)",
111			`,
112			expect: "__SBOX_SANDBOX_DIR__/tools/out/bin/tool > __SBOX_SANDBOX_DIR__/out/out",
113		},
114		{
115			name: "empty location tool2",
116			prop: `
117				tools: [":tool"],
118				out: ["out"],
119				cmd: "$(location) > $(out)",
120			`,
121			expect: "__SBOX_SANDBOX_DIR__/tools/out/bin/tool > __SBOX_SANDBOX_DIR__/out/out",
122		},
123		{
124			name: "empty location tool file",
125			prop: `
126				tool_files: ["tool_file1"],
127				out: ["out"],
128				cmd: "$(location) > $(out)",
129			`,
130			expect: "__SBOX_SANDBOX_DIR__/tools/src/tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
131		},
132		{
133			name: "empty location tool file fg",
134			prop: `
135				tool_files: [":1tool_file"],
136				out: ["out"],
137				cmd: "$(location) > $(out)",
138			`,
139			expect: "__SBOX_SANDBOX_DIR__/tools/src/tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
140		},
141		{
142			name: "empty location tool and tool file",
143			prop: `
144				tools: ["tool"],
145				tool_files: ["tool_file1"],
146				out: ["out"],
147				cmd: "$(location) > $(out)",
148			`,
149			expect: "__SBOX_SANDBOX_DIR__/tools/out/bin/tool > __SBOX_SANDBOX_DIR__/out/out",
150		},
151		{
152			name: "tool",
153			prop: `
154				tools: ["tool"],
155				out: ["out"],
156				cmd: "$(location tool) > $(out)",
157			`,
158			expect: "__SBOX_SANDBOX_DIR__/tools/out/bin/tool > __SBOX_SANDBOX_DIR__/out/out",
159		},
160		{
161			name: "tool2",
162			prop: `
163				tools: [":tool"],
164				out: ["out"],
165				cmd: "$(location :tool) > $(out)",
166			`,
167			expect: "__SBOX_SANDBOX_DIR__/tools/out/bin/tool > __SBOX_SANDBOX_DIR__/out/out",
168		},
169		{
170			name: "tool file",
171			prop: `
172				tool_files: ["tool_file1"],
173				out: ["out"],
174				cmd: "$(location tool_file1) > $(out)",
175			`,
176			expect: "__SBOX_SANDBOX_DIR__/tools/src/tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
177		},
178		{
179			name: "tool file fg",
180			prop: `
181				tool_files: [":1tool_file"],
182				out: ["out"],
183				cmd: "$(location :1tool_file) > $(out)",
184			`,
185			expect: "__SBOX_SANDBOX_DIR__/tools/src/tool_file1 > __SBOX_SANDBOX_DIR__/out/out",
186		},
187		{
188			name: "tool files",
189			prop: `
190				tool_files: [":tool_files"],
191				out: ["out"],
192				cmd: "$(locations :tool_files) > $(out)",
193			`,
194			expect: "__SBOX_SANDBOX_DIR__/tools/src/tool_file1 __SBOX_SANDBOX_DIR__/tools/src/tool_file2 > __SBOX_SANDBOX_DIR__/out/out",
195		},
196		{
197			name: "in1",
198			prop: `
199				srcs: ["in1"],
200				out: ["out"],
201				cmd: "cat $(in) > $(out)",
202			`,
203			expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
204		},
205		{
206			name: "in1 fg",
207			prop: `
208				srcs: [":1in"],
209				out: ["out"],
210				cmd: "cat $(in) > $(out)",
211			`,
212			expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
213		},
214		{
215			name: "ins",
216			prop: `
217				srcs: ["in1", "in2"],
218				out: ["out"],
219				cmd: "cat $(in) > $(out)",
220			`,
221			expect: "cat in1 in2 > __SBOX_SANDBOX_DIR__/out/out",
222		},
223		{
224			name: "ins fg",
225			prop: `
226				srcs: [":ins"],
227				out: ["out"],
228				cmd: "cat $(in) > $(out)",
229			`,
230			expect: "cat in1 in2 > __SBOX_SANDBOX_DIR__/out/out",
231		},
232		{
233			name: "location in1",
234			prop: `
235				srcs: ["in1"],
236				out: ["out"],
237				cmd: "cat $(location in1) > $(out)",
238			`,
239			expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
240		},
241		{
242			name: "location in1 fg",
243			prop: `
244				srcs: [":1in"],
245				out: ["out"],
246				cmd: "cat $(location :1in) > $(out)",
247			`,
248			expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
249		},
250		{
251			name: "location ins",
252			prop: `
253				srcs: ["in1", "in2"],
254				out: ["out"],
255				cmd: "cat $(location in1) > $(out)",
256			`,
257			expect: "cat in1 > __SBOX_SANDBOX_DIR__/out/out",
258		},
259		{
260			name: "location ins fg",
261			prop: `
262				srcs: [":ins"],
263				out: ["out"],
264				cmd: "cat $(locations :ins) > $(out)",
265			`,
266			expect: "cat in1 in2 > __SBOX_SANDBOX_DIR__/out/out",
267		},
268		{
269			name: "outs",
270			prop: `
271				out: ["out", "out2"],
272				cmd: "echo foo > $(out)",
273			`,
274			expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out __SBOX_SANDBOX_DIR__/out/out2",
275		},
276		{
277			name: "location out",
278			prop: `
279				out: ["out", "out2"],
280				cmd: "echo foo > $(location out2)",
281			`,
282			expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out2",
283		},
284		{
285			name: "depfile",
286			prop: `
287				out: ["out"],
288				depfile: true,
289				cmd: "echo foo > $(out) && touch $(depfile)",
290			`,
291			expect: "echo foo > __SBOX_SANDBOX_DIR__/out/out && touch __SBOX_DEPFILE__",
292		},
293		{
294			name: "gendir",
295			prop: `
296				out: ["out"],
297				cmd: "echo foo > $(genDir)/foo && cp $(genDir)/foo $(out)",
298			`,
299			expect: "echo foo > __SBOX_SANDBOX_DIR__/out/foo && cp __SBOX_SANDBOX_DIR__/out/foo __SBOX_SANDBOX_DIR__/out/out",
300		},
301		{
302			name: "$",
303			prop: `
304				out: ["out"],
305				cmd: "echo $$ > $(out)",
306			`,
307			expect: "echo $ > __SBOX_SANDBOX_DIR__/out/out",
308		},
309
310		{
311			name: "error empty location",
312			prop: `
313				out: ["out"],
314				cmd: "$(location) > $(out)",
315			`,
316			err: "at least one `tools` or `tool_files` is required if $(location) is used",
317		},
318		{
319			name: "error empty location no files",
320			prop: `
321				tool_files: [":empty"],
322				out: ["out"],
323				cmd: "$(location) > $(out)",
324			`,
325			err: `default label ":empty" has no files`,
326		},
327		{
328			name: "error empty location multiple files",
329			prop: `
330				tool_files: [":tool_files"],
331				out: ["out"],
332				cmd: "$(location) > $(out)",
333			`,
334			err: `default label ":tool_files" has multiple files`,
335		},
336		{
337			name: "error location",
338			prop: `
339				out: ["out"],
340				cmd: "echo foo > $(location missing)",
341			`,
342			err: `unknown location label "missing"`,
343		},
344		{
345			name: "error locations",
346			prop: `
347					out: ["out"],
348					cmd: "echo foo > $(locations missing)",
349			`,
350			err: `unknown locations label "missing"`,
351		},
352		{
353			name: "error location no files",
354			prop: `
355					out: ["out"],
356					srcs: [":empty"],
357					cmd: "echo $(location :empty) > $(out)",
358			`,
359			err: `label ":empty" has no files`,
360		},
361		{
362			name: "error locations no files",
363			prop: `
364					out: ["out"],
365					srcs: [":empty"],
366					cmd: "echo $(locations :empty) > $(out)",
367			`,
368			err: `label ":empty" has no files`,
369		},
370		{
371			name: "error location multiple files",
372			prop: `
373					out: ["out"],
374					srcs: [":ins"],
375					cmd: "echo $(location :ins) > $(out)",
376			`,
377			err: `label ":ins" has multiple files`,
378		},
379		{
380			name: "error variable",
381			prop: `
382					out: ["out"],
383					srcs: ["in1"],
384					cmd: "echo $(foo) > $(out)",
385			`,
386			err: `unknown variable '$(foo)'`,
387		},
388		{
389			name: "error depfile",
390			prop: `
391				out: ["out"],
392				cmd: "echo foo > $(out) && touch $(depfile)",
393			`,
394			err: "$(depfile) used without depfile property",
395		},
396		{
397			name: "error no depfile",
398			prop: `
399				out: ["out"],
400				depfile: true,
401				cmd: "echo foo > $(out)",
402			`,
403			err: "specified depfile=true but did not include a reference to '${depfile}' in cmd",
404		},
405		{
406			name: "error no out",
407			prop: `
408				cmd: "echo foo > $(out)",
409			`,
410			err: "must have at least one output file",
411		},
412		{
413			name: "srcs allow missing dependencies",
414			prop: `
415				srcs: [":missing"],
416				out: ["out"],
417				cmd: "cat $(location :missing) > $(out)",
418			`,
419
420			allowMissingDependencies: true,
421
422			expect: "cat ***missing srcs :missing*** > __SBOX_SANDBOX_DIR__/out/out",
423		},
424		{
425			name: "tool allow missing dependencies",
426			prop: `
427				tools: [":missing"],
428				out: ["out"],
429				cmd: "$(location :missing) > $(out)",
430			`,
431
432			allowMissingDependencies: true,
433
434			expect: "***missing tool :missing*** > __SBOX_SANDBOX_DIR__/out/out",
435		},
436	}
437
438	for _, test := range testcases {
439		t.Run(test.name, func(t *testing.T) {
440			bp := "genrule {\n"
441			bp += "name: \"gen\",\n"
442			bp += test.prop
443			bp += "}\n"
444
445			var expectedErrors []string
446			if test.err != "" {
447				expectedErrors = append(expectedErrors, regexp.QuoteMeta(test.err))
448			}
449
450			result := android.GroupFixturePreparers(
451				prepareForGenRuleTest,
452				android.FixtureModifyProductVariables(func(variables android.FixtureProductVariables) {
453					variables.Allow_missing_dependencies = proptools.BoolPtr(test.allowMissingDependencies)
454				}),
455				android.FixtureModifyContext(func(ctx *android.TestContext) {
456					ctx.SetAllowMissingDependencies(test.allowMissingDependencies)
457				}),
458			).
459				ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(expectedErrors)).
460				RunTestWithBp(t, testGenruleBp()+bp)
461
462			if expectedErrors != nil {
463				return
464			}
465
466			gen := result.Module("gen", "").(*Module)
467			android.AssertStringEquals(t, "raw commands", test.expect, gen.rawCommands[0])
468		})
469	}
470}
471
472func TestGenruleHashInputs(t *testing.T) {
473
474	// The basic idea here is to verify that the sbox command (which is
475	// in the Command field of the generate rule) contains a hash of the
476	// inputs, but only if $(in) is not referenced in the genrule cmd
477	// property.
478
479	// By including a hash of the inputs, we cause the rule to re-run if
480	// the list of inputs changes (because the sbox command changes).
481
482	// However, if the genrule cmd property already contains $(in), then
483	// the dependency is already expressed, so we don't need to include the
484	// hash in that case.
485
486	bp := `
487			genrule {
488				name: "hash0",
489				srcs: ["in1.txt", "in2.txt"],
490				out: ["out"],
491				cmd: "echo foo > $(out)",
492			}
493			genrule {
494				name: "hash1",
495				srcs: ["*.txt"],
496				out: ["out"],
497				cmd: "echo bar > $(out)",
498			}
499			genrule {
500				name: "hash2",
501				srcs: ["*.txt"],
502				out: ["out"],
503				cmd: "echo $(in) > $(out)",
504			}
505		`
506	testcases := []struct {
507		name         string
508		expectedHash string
509	}{
510		{
511			name: "hash0",
512			// sha256 value obtained from: echo -en 'in1.txt\nin2.txt' | sha256sum
513			expectedHash: "18da75b9b1cc74b09e365b4ca2e321b5d618f438cc632b387ad9dc2ab4b20e9d",
514		},
515		{
516			name: "hash1",
517			// sha256 value obtained from: echo -en 'in1.txt\nin2.txt\nin3.txt' | sha256sum
518			expectedHash: "a38d432a4b19df93140e1f1fe26c97ff0387dae01fe506412b47208f0595fb45",
519		},
520		{
521			name: "hash2",
522			// sha256 value obtained from: echo -en 'in1.txt\nin2.txt\nin3.txt' | sha256sum
523			expectedHash: "a38d432a4b19df93140e1f1fe26c97ff0387dae01fe506412b47208f0595fb45",
524		},
525	}
526
527	result := prepareForGenRuleTest.RunTestWithBp(t, testGenruleBp()+bp)
528
529	for _, test := range testcases {
530		t.Run(test.name, func(t *testing.T) {
531			gen := result.ModuleForTests(test.name, "")
532			manifest := android.RuleBuilderSboxProtoForTests(t, gen.Output("genrule.sbox.textproto"))
533			hash := manifest.Commands[0].GetInputHash()
534
535			android.AssertStringEquals(t, "hash", test.expectedHash, hash)
536		})
537	}
538}
539
540func TestGenSrcs(t *testing.T) {
541	testcases := []struct {
542		name string
543		prop string
544
545		allowMissingDependencies bool
546
547		err   string
548		cmds  []string
549		deps  []string
550		files []string
551	}{
552		{
553			name: "gensrcs",
554			prop: `
555				tools: ["tool"],
556				srcs: ["in1.txt", "in2.txt"],
557				cmd: "$(location) $(in) > $(out)",
558			`,
559			cmds: []string{
560				"bash -c '__SBOX_SANDBOX_DIR__/tools/out/bin/tool in1.txt > __SBOX_SANDBOX_DIR__/out/in1.h' && bash -c '__SBOX_SANDBOX_DIR__/tools/out/bin/tool in2.txt > __SBOX_SANDBOX_DIR__/out/in2.h'",
561			},
562			deps: []string{
563				"out/soong/.intermediates/gen/gen/gensrcs/in1.h",
564				"out/soong/.intermediates/gen/gen/gensrcs/in2.h",
565			},
566			files: []string{
567				"out/soong/.intermediates/gen/gen/gensrcs/in1.h",
568				"out/soong/.intermediates/gen/gen/gensrcs/in2.h",
569			},
570		},
571		{
572			name: "shards",
573			prop: `
574				tools: ["tool"],
575				srcs: ["in1.txt", "in2.txt", "in3.txt"],
576				cmd: "$(location) $(in) > $(out)",
577				shard_size: 2,
578			`,
579			cmds: []string{
580				"bash -c '__SBOX_SANDBOX_DIR__/tools/out/bin/tool in1.txt > __SBOX_SANDBOX_DIR__/out/in1.h' && bash -c '__SBOX_SANDBOX_DIR__/tools/out/bin/tool in2.txt > __SBOX_SANDBOX_DIR__/out/in2.h'",
581				"bash -c '__SBOX_SANDBOX_DIR__/tools/out/bin/tool in3.txt > __SBOX_SANDBOX_DIR__/out/in3.h'",
582			},
583			deps: []string{
584				"out/soong/.intermediates/gen/gen/gensrcs/in1.h",
585				"out/soong/.intermediates/gen/gen/gensrcs/in2.h",
586				"out/soong/.intermediates/gen/gen/gensrcs/in3.h",
587			},
588			files: []string{
589				"out/soong/.intermediates/gen/gen/gensrcs/in1.h",
590				"out/soong/.intermediates/gen/gen/gensrcs/in2.h",
591				"out/soong/.intermediates/gen/gen/gensrcs/in3.h",
592			},
593		},
594	}
595
596	for _, test := range testcases {
597		t.Run(test.name, func(t *testing.T) {
598			bp := "gensrcs {\n"
599			bp += `name: "gen",` + "\n"
600			bp += `output_extension: "h",` + "\n"
601			bp += test.prop
602			bp += "}\n"
603
604			var expectedErrors []string
605			if test.err != "" {
606				expectedErrors = append(expectedErrors, regexp.QuoteMeta(test.err))
607			}
608
609			result := prepareForGenRuleTest.
610				ExtendWithErrorHandler(android.FixtureExpectsAllErrorsToMatchAPattern(expectedErrors)).
611				RunTestWithBp(t, testGenruleBp()+bp)
612
613			if expectedErrors != nil {
614				return
615			}
616
617			gen := result.Module("gen", "").(*Module)
618			android.AssertDeepEquals(t, "cmd", test.cmds, gen.rawCommands)
619
620			android.AssertPathsRelativeToTopEquals(t, "deps", test.deps, gen.outputDeps)
621
622			android.AssertPathsRelativeToTopEquals(t, "files", test.files, gen.outputFiles)
623		})
624	}
625}
626
627func TestGenruleDefaults(t *testing.T) {
628	bp := `
629				genrule_defaults {
630					name: "gen_defaults1",
631					cmd: "cp $(in) $(out)",
632				}
633
634				genrule_defaults {
635					name: "gen_defaults2",
636					srcs: ["in1"],
637				}
638
639				genrule {
640					name: "gen",
641					out: ["out"],
642					defaults: ["gen_defaults1", "gen_defaults2"],
643				}
644			`
645
646	result := prepareForGenRuleTest.RunTestWithBp(t, testGenruleBp()+bp)
647
648	gen := result.Module("gen", "").(*Module)
649
650	expectedCmd := "cp in1 __SBOX_SANDBOX_DIR__/out/out"
651	android.AssertStringEquals(t, "cmd", expectedCmd, gen.rawCommands[0])
652
653	expectedSrcs := []string{"in1"}
654	android.AssertDeepEquals(t, "srcs", expectedSrcs, gen.properties.Srcs)
655}
656
657func TestGenruleAllowMissingDependencies(t *testing.T) {
658	bp := `
659		output {
660			name: "disabled",
661			enabled: false,
662		}
663
664		genrule {
665			name: "gen",
666			srcs: [
667				":disabled",
668			],
669			out: ["out"],
670			cmd: "cat $(in) > $(out)",
671		}
672       `
673	result := android.GroupFixturePreparers(
674		prepareForGenRuleTest,
675		android.FixtureModifyConfigAndContext(
676			func(config android.Config, ctx *android.TestContext) {
677				config.TestProductVariables.Allow_missing_dependencies = proptools.BoolPtr(true)
678				ctx.SetAllowMissingDependencies(true)
679			})).RunTestWithBp(t, bp)
680
681	gen := result.ModuleForTests("gen", "").Output("out")
682	if gen.Rule != android.ErrorRule {
683		t.Errorf("Expected missing dependency error rule for gen, got %q", gen.Rule.String())
684	}
685}
686
687func TestGenruleWithBazel(t *testing.T) {
688	bp := `
689		genrule {
690				name: "foo",
691				out: ["one.txt", "two.txt"],
692				bazel_module: { label: "//foo/bar:bar" },
693		}
694	`
695
696	result := android.GroupFixturePreparers(
697		prepareForGenRuleTest, android.FixtureModifyConfig(func(config android.Config) {
698			config.BazelContext = android.MockBazelContext{
699				OutputBaseDir: "outputbase",
700				LabelToOutputFiles: map[string][]string{
701					"//foo/bar:bar": []string{"bazelone.txt", "bazeltwo.txt"}}}
702		})).RunTestWithBp(t, testGenruleBp()+bp)
703
704	gen := result.Module("foo", "").(*Module)
705
706	expectedOutputFiles := []string{"outputbase/execroot/__main__/bazelone.txt",
707		"outputbase/execroot/__main__/bazeltwo.txt"}
708	android.AssertDeepEquals(t, "output files", expectedOutputFiles, gen.outputFiles.Strings())
709	android.AssertDeepEquals(t, "output deps", expectedOutputFiles, gen.outputDeps.Strings())
710}
711
712type testTool struct {
713	android.ModuleBase
714	outputFile android.Path
715}
716
717func toolFactory() android.Module {
718	module := &testTool{}
719	android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst)
720	return module
721}
722
723func (t *testTool) GenerateAndroidBuildActions(ctx android.ModuleContext) {
724	t.outputFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "bin"), ctx.ModuleName(), android.PathForOutput(ctx, ctx.ModuleName()))
725}
726
727func (t *testTool) HostToolPath() android.OptionalPath {
728	return android.OptionalPathForPath(t.outputFile)
729}
730
731var _ android.HostToolProvider = (*testTool)(nil)
732
733type testOutputProducer struct {
734	android.ModuleBase
735	outputFile android.Path
736}
737
738func outputProducerFactory() android.Module {
739	module := &testOutputProducer{}
740	android.InitAndroidArchModule(module, android.HostSupported, android.MultilibFirst)
741	return module
742}
743
744func (t *testOutputProducer) GenerateAndroidBuildActions(ctx android.ModuleContext) {
745	t.outputFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "bin"), ctx.ModuleName(), android.PathForOutput(ctx, ctx.ModuleName()))
746}
747
748func (t *testOutputProducer) OutputFiles(tag string) (android.Paths, error) {
749	return android.Paths{t.outputFile}, nil
750}
751
752var _ android.OutputFileProducer = (*testOutputProducer)(nil)
753