1// Copyright 2020 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package build
16
17// This file contains the functionality to upload data from one location to
18// another.
19
20import (
21	"io/ioutil"
22	"os"
23	"path/filepath"
24	"time"
25
26	"android/soong/ui/metrics"
27	"github.com/golang/protobuf/proto"
28
29	upload_proto "android/soong/ui/metrics/upload_proto"
30)
31
32const (
33	// Used to generate a raw protobuf file that contains information
34	// of the list of metrics files from host to destination storage.
35	uploadPbFilename = ".uploader.pb"
36)
37
38var (
39	// For testing purpose.
40	tmpDir = ioutil.TempDir
41)
42
43// pruneMetricsFiles iterates the list of paths, checking if a path exist.
44// If a path is a file, it is added to the return list. If the path is a
45// directory, a recursive call is made to add the children files of the
46// path.
47func pruneMetricsFiles(paths []string) []string {
48	var metricsFiles []string
49	for _, p := range paths {
50		fi, err := os.Stat(p)
51		// Some paths passed may not exist. For example, build errors protobuf
52		// file may not exist since the build was successful.
53		if err != nil {
54			continue
55		}
56
57		if fi.IsDir() {
58			if l, err := ioutil.ReadDir(p); err == nil {
59				files := make([]string, 0, len(l))
60				for _, fi := range l {
61					files = append(files, filepath.Join(p, fi.Name()))
62				}
63				metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
64			}
65		} else {
66			metricsFiles = append(metricsFiles, p)
67		}
68	}
69	return metricsFiles
70}
71
72// UploadMetrics uploads a set of metrics files to a server for analysis. An
73// uploader full path is specified in ANDROID_ENABLE_METRICS_UPLOAD environment
74// variable in order to upload the set of metrics files. The metrics files are
75// first copied to a temporary directory and the uploader is then executed in
76// the background to allow the user/system to continue working. Soong communicates
77// to the uploader through the upload_proto raw protobuf file.
78func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) {
79	ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
80	defer ctx.EndTrace()
81
82	uploader := config.MetricsUploaderApp()
83	if uploader == "" {
84		// If the uploader path was not specified, no metrics shall be uploaded.
85		return
86	}
87
88	// Several of the files might be directories.
89	metricsFiles := pruneMetricsFiles(paths)
90	if len(metricsFiles) == 0 {
91		return
92	}
93
94	// The temporary directory cannot be deleted as the metrics uploader is started
95	// in the background and requires to exist until the operation is done. The
96	// uploader can delete the directory as it is specified in the upload proto.
97	tmpDir, err := tmpDir("", "upload_metrics")
98	if err != nil {
99		ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err)
100	}
101
102	for i, src := range metricsFiles {
103		dst := filepath.Join(tmpDir, filepath.Base(src))
104		if _, err := copyFile(src, dst); err != nil {
105			ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err)
106		}
107		metricsFiles[i] = dst
108	}
109
110	// For platform builds, the branch and target name is hardcoded to specific
111	// values for later extraction of the metrics in the data metrics pipeline.
112	data, err := proto.Marshal(&upload_proto.Upload{
113		CreationTimestampMs:   proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))),
114		CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
115		BranchName:            proto.String("developer-metrics"),
116		TargetName:            proto.String("platform-build-systems-metrics"),
117		MetricsFiles:          metricsFiles,
118		DirectoriesToDelete:   []string{tmpDir},
119	})
120	if err != nil {
121		ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
122	}
123
124	pbFile := filepath.Join(tmpDir, uploadPbFilename)
125	if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
126		ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
127	}
128
129	// Start the uploader in the background as it takes several milliseconds to start the uploader
130	// and prepare the metrics for upload. This affects small shell commands like "lunch".
131	cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile)
132	if simpleOutput {
133		cmd.RunOrFatal()
134	} else {
135		cmd.RunAndStreamOrFatal()
136	}
137}
138