1// Copyright 2020 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5package main
6
7import (
8	"context"
9	"encoding/json"
10	"flag"
11	"fmt"
12	"io/ioutil"
13	"os"
14	"path/filepath"
15	"strconv"
16
17	"go.skia.org/infra/go/common"
18	"go.skia.org/infra/go/exec"
19	"go.skia.org/infra/go/httputils"
20	"go.skia.org/infra/go/skerr"
21	"go.skia.org/infra/task_driver/go/lib/os_steps"
22	"go.skia.org/infra/task_driver/go/td"
23)
24
25func main() {
26	var (
27		// Required properties for this task.
28		builtPath       = flag.String("built_path", "", "The directory where the built wasm/js code will be.")
29		gitCommit       = flag.String("git_commit", "", "The commit at which we are testing.")
30		goldCtlPath     = flag.String("gold_ctl_path", "", "Path to the goldctl binary")
31		goldKeys        = common.NewMultiStringFlag("gold_key", nil, "The keys that will tag this data")
32		nodeBinPath     = flag.String("node_bin_path", "", "Path to the node bin directory (should have npm also). This directory *must* be on the PATH when this executable is called, otherwise, the wrong node or npm version may be found (e.g. the one on the system), even if we are explicitly calling npm with the absolute path.")
33		projectID       = flag.String("project_id", "", "ID of the Google Cloud project.")
34		resourcePath    = flag.String("resource_path", "", "The directory housing the images, fonts, and other assets used by tests.")
35		taskID          = flag.String("task_id", "", "task id this data was generated on")
36		taskName        = flag.String("task_name", "", "Name of the task.")
37		testHarnessPath = flag.String("test_harness_path", "", "Path to test harness folder (tools/run-wasm-gm-tests)")
38		webGLVersion    = flag.Int("webgl_version", 2, "The version of web gl to use. 0 means CPU")
39		workPath        = flag.String("work_path", "", "The directory to use to store temporary files (e.g. pngs and JSON)")
40
41		// Provided for tryjobs
42		changelistID = flag.String("changelist_id", "", "The id the Gerrit CL. Omit for primary branch.")
43		tryjobID     = flag.String("tryjob_id", "", "The id of the Buildbucket job for tryjobs. Omit for primary branch.")
44		// Because we pass in patchset_order via a placeholder, it can be empty string. As such, we
45		// cannot use flag.Int, because that errors on "" being passed in.
46		patchsetOrder = flag.String("patchset_order", "0", "Represents if this is the nth patchset")
47
48		// Debugging flags.
49		local              = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
50		outputSteps        = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
51		serviceAccountPath = flag.String("service_account_path", "", "Used in local mode for authentication. Non-local mode uses Luci config.")
52	)
53
54	// Setup.
55	ctx := td.StartRun(projectID, taskID, taskName, outputSteps, local)
56	defer td.EndRun(ctx)
57
58	builtAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *builtPath, "built_path")
59	goldctlAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *goldCtlPath, "gold_ctl_path")
60	nodeBinAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *nodeBinPath, "node_bin_path")
61	resourceAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *resourcePath, "resource_path")
62	testHarnessAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *testHarnessPath, "test_harness_path")
63	workAbsPath := td.MustGetAbsolutePathOfFlag(ctx, *workPath, "work_path")
64
65	goldctlWorkPath := filepath.Join(workAbsPath, "goldctl")
66	if err := os_steps.MkdirAll(ctx, goldctlWorkPath); err != nil {
67		td.Fatal(ctx, err)
68	}
69	testsWorkPath := filepath.Join(workAbsPath, "tests")
70	if err := os_steps.MkdirAll(ctx, testsWorkPath); err != nil {
71		td.Fatal(ctx, err)
72	}
73
74	patchset := 0
75	if *patchsetOrder != "" {
76		p, err := strconv.Atoi(*patchsetOrder)
77		if err != nil {
78			td.Fatalf(ctx, "Invalid patchset_order %q", *patchsetOrder)
79		}
80		patchset = p
81	}
82
83	keys := *goldKeys
84	switch *webGLVersion {
85	case 0:
86		keys = append(keys, "cpu_or_gpu:CPU")
87	case 1:
88		keys = append(keys, "cpu_or_gpu:GPU", "extra_config:WebGL1")
89	case 2:
90		keys = append(keys, "cpu_or_gpu:GPU", "extra_config:WebGL2")
91	default:
92		td.Fatalf(ctx, "Invalid value for webgl_version, must be 0, 1, 2 got %d", *webGLVersion)
93	}
94
95	// initialize goldctl
96	if err := setupGoldctl(ctx, *local, *gitCommit, *changelistID, *tryjobID, goldctlAbsPath, goldctlWorkPath,
97		*serviceAccountPath, keys, patchset); err != nil {
98		td.Fatal(ctx, err)
99	}
100
101	if err := downloadKnownHashes(ctx, testsWorkPath); err != nil {
102		td.Fatal(ctx, err)
103	}
104	if err := setupTests(ctx, nodeBinAbsPath, testHarnessAbsPath); err != nil {
105		td.Fatal(ctx, skerr.Wrap(err))
106	}
107	// Run puppeteer tests. The input is a list of known hashes. The output will be a JSON array and
108	// any new images to be written to disk in the testsWorkPath. See WriteToDisk in DM for how that
109	// is done on the C++ side.
110	if err := runTests(ctx, builtAbsPath, nodeBinAbsPath, resourceAbsPath, testHarnessAbsPath, testsWorkPath, *webGLVersion); err != nil {
111		td.Fatal(ctx, err)
112	}
113
114	// Parse JSON and call goldctl imgtest add them.
115	if err := processTestData(ctx, testsWorkPath, goldctlAbsPath, goldctlWorkPath); err != nil {
116		td.Fatal(ctx, err)
117	}
118
119	// call goldctl finalize to upload stuff.
120	if err := finalizeGoldctl(ctx, goldctlAbsPath, goldctlWorkPath); err != nil {
121		td.Fatal(ctx, err)
122	}
123}
124
125func setupGoldctl(ctx context.Context, local bool, gitCommit, gerritCLID, tryjobID, goldctlPath, workPath, serviceAccountPath string, keys []string, psOrder int) error {
126	ctx = td.StartStep(ctx, td.Props("setup goldctl").Infra())
127	defer td.EndStep(ctx)
128
129	args := []string{goldctlPath, "auth", "--work-dir", workPath}
130	if !local {
131		args = append(args, "--luci")
132	} else {
133		// When testing locally, it can also be handy to add in --dry-run here.
134		args = append(args, "--service-account", serviceAccountPath)
135	}
136
137	if _, err := exec.RunCwd(ctx, workPath, args...); err != nil {
138		return td.FailStep(ctx, skerr.Wrapf(err, "running %s", args))
139	}
140
141	args = []string{
142		goldctlPath, "imgtest", "init", "--work-dir", workPath, "--instance", "skia", "--corpus", "gm",
143		"--commit", gitCommit,
144	}
145	if gerritCLID != "" {
146		ps := strconv.Itoa(psOrder)
147		args = append(args, "--crs", "gerrit", "--changelist", gerritCLID, "--patchset", ps,
148			"--cis", "buildbucket", "--jobid", tryjobID)
149	}
150
151	for _, key := range keys {
152		args = append(args, "--key", key)
153	}
154
155	if _, err := exec.RunCwd(ctx, workPath, args...); err != nil {
156		return td.FailStep(ctx, skerr.Wrapf(err, "running %s", args))
157	}
158	return nil
159}
160
161const knownHashesURL = "https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt"
162
163// downloadKnownHashes downloads the known hashes from Gold and stores it as a text file in
164// workPath/hashes.txt
165func downloadKnownHashes(ctx context.Context, workPath string) error {
166	ctx = td.StartStep(ctx, td.Props("download known hashes").Infra())
167	defer td.EndStep(ctx)
168
169	client := httputils.DefaultClientConfig().With2xxOnly().Client()
170	resp, err := client.Get(knownHashesURL)
171	if err != nil {
172		return td.FailStep(ctx, skerr.Wrapf(err, "downloading known hashes"))
173	}
174	defer resp.Body.Close()
175	data, err := ioutil.ReadAll(resp.Body)
176	if err != nil {
177		return td.FailStep(ctx, skerr.Wrapf(err, "reading known hashes"))
178	}
179	return os_steps.WriteFile(ctx, filepath.Join(workPath, "hashes.txt"), data, 0666)
180}
181
182func setupTests(ctx context.Context, nodeBinPath string, testHarnessPath string) error {
183	ctx = td.StartStep(ctx, td.Props("setup npm").Infra())
184	defer td.EndStep(ctx)
185
186	if _, err := exec.RunCwd(ctx, testHarnessPath, filepath.Join(nodeBinPath, "npm"), "ci"); err != nil {
187		return td.FailStep(ctx, skerr.Wrap(err))
188	}
189	return nil
190}
191
192func runTests(ctx context.Context, builtPath, nodeBinPath, resourcePath, testHarnessPath, workPath string, webglVersion int) error {
193	ctx = td.StartStep(ctx, td.Props("run GMs and unit tests"))
194	defer td.EndStep(ctx)
195
196	err := td.Do(ctx, td.Props("Run GMs and Unit Tests"), func(ctx context.Context) error {
197		args := []string{filepath.Join(nodeBinPath, "node"),
198			"run-wasm-gm-tests",
199			"--js_file", filepath.Join(builtPath, "wasm_gm_tests.js"),
200			"--wasm_file", filepath.Join(builtPath, "wasm_gm_tests.wasm"),
201			"--known_hashes", filepath.Join(workPath, "hashes.txt"),
202			"--use_gpu", // TODO(kjlubick) use webglVersion and account for CPU
203			"--output", workPath,
204			"--resources", resourcePath,
205			"--timeout", "180", // seconds per batch of 50 tests.
206		}
207
208		_, err := exec.RunCwd(ctx, testHarnessPath, args...)
209		if err != nil {
210			return skerr.Wrap(err)
211		}
212		return nil
213	})
214	if err != nil {
215		return td.FailStep(ctx, skerr.Wrap(err))
216	}
217	return nil
218}
219
220type goldResult struct {
221	TestName string `json:"name"`
222	MD5Hash  string `json:"digest"`
223}
224
225func processTestData(ctx context.Context, testOutputPath, goldctlPath, goldctlWorkPath string) error {
226	ctx = td.StartStep(ctx, td.Props("process test data").Infra())
227	defer td.EndStep(ctx)
228
229	// Read in the file, process it as []goldResult
230	var results []goldResult
231	resultFile := filepath.Join(testOutputPath, "gold_results.json")
232
233	err := td.Do(ctx, td.Props("Load results from "+resultFile), func(ctx context.Context) error {
234		b, err := os_steps.ReadFile(ctx, resultFile)
235		if err != nil {
236			return skerr.Wrap(err)
237		}
238		if err := json.Unmarshal(b, &results); err != nil {
239			return skerr.Wrap(err)
240		}
241		return nil
242	})
243	if err != nil {
244		return td.FailStep(ctx, skerr.Wrap(err))
245	}
246
247	err = td.Do(ctx, td.Props(fmt.Sprintf("Call goldtl on %d results", len(results))), func(ctx context.Context) error {
248		for _, result := range results {
249			// These args are the same regardless of if we need to upload the png file or not.
250			args := []string{goldctlPath, "imgtest", "add", "--work-dir", goldctlWorkPath,
251				"--test-name", result.TestName, "--png-digest", result.MD5Hash}
252			// check to see if there's an image we need to upload
253			potentialPNGFile := filepath.Join(testOutputPath, result.MD5Hash+".png")
254			_, err := os_steps.Stat(ctx, potentialPNGFile)
255			if os.IsNotExist(err) {
256				// PNG was not produced, we assume it is already uploaded to Gold and just say the digest
257				// we produced.
258				_, err = exec.RunCwd(ctx, goldctlWorkPath, args...)
259				if err != nil {
260					return skerr.Wrapf(err, "reporting result %#v to goldctl", result)
261				}
262				continue
263			} else if err != nil {
264				return skerr.Wrapf(err, "reading %s", potentialPNGFile)
265			}
266			// call goldctl with the png file
267			args = append(args, "--png-file", potentialPNGFile)
268			_, err = exec.RunCwd(ctx, goldctlWorkPath, args...)
269			if err != nil {
270				return skerr.Wrapf(err, "reporting result %#v to goldctl", result)
271			}
272		}
273		return nil
274	})
275	if err != nil {
276		return td.FailStep(ctx, skerr.Wrap(err))
277	}
278	return nil
279}
280
281func finalizeGoldctl(ctx context.Context, goldctlPath, workPath string) error {
282	ctx = td.StartStep(ctx, td.Props("finalize goldctl data").Infra())
283	defer td.EndStep(ctx)
284
285	_, err := exec.RunCwd(ctx, workPath, goldctlPath, "imgtest", "finalize", "--work-dir", workPath)
286	if err != nil {
287		return skerr.Wrapf(err, "Finalizing goldctl")
288	}
289	return nil
290}
291