1# Copyright 2020 Google Inc. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# 15################################################################################ 16"""Unit tests for Cloud Function sync, which syncs the list of github projects 17and uploads them to the Cloud Datastore.""" 18 19import os 20import sys 21import unittest 22 23from google.cloud import ndb 24 25sys.path.append(os.path.dirname(__file__)) 26# pylint: disable=wrong-import-position 27 28from datastore_entities import Project 29from project_sync import get_github_creds 30from project_sync import get_projects 31from project_sync import ProjectMetadata 32from project_sync import sync_projects 33import test_utils 34 35# pylint: disable=no-member 36 37 38# pylint: disable=too-few-public-methods 39class Repository: 40 """Mocking Github Repository.""" 41 42 def __init__(self, name, file_type, path, contents=None): 43 self.contents = contents or [] 44 self.name = name 45 self.type = file_type 46 self.path = path 47 self.decoded_content = b"name: test" 48 49 def get_contents(self, path): 50 """"Get contents of repository.""" 51 if self.path == path: 52 return self.contents 53 54 for content_file in self.contents: 55 if content_file.path == path: 56 return content_file.contents 57 58 return None 59 60 def set_yaml_contents(self, decoded_content): 61 """Set yaml_contents.""" 62 self.decoded_content = decoded_content 63 64 65class CloudSchedulerClient: 66 """Mocking cloud scheduler client.""" 67 68 def __init__(self): 69 self.schedulers = [] 70 71 # pylint: disable=no-self-use 72 def location_path(self, project_id, location_id): 73 """Return project path.""" 74 return 'projects/{}/location/{}'.format(project_id, location_id) 75 76 def create_job(self, parent, job): 77 """Simulate create job.""" 78 del parent 79 self.schedulers.append(job) 80 81 # pylint: disable=no-self-use 82 def job_path(self, project_id, location_id, name): 83 """Return job path.""" 84 return 'projects/{}/location/{}/jobs/{}'.format(project_id, location_id, 85 name) 86 87 def delete_job(self, name): 88 """Simulate delete jobs.""" 89 for job in self.schedulers: 90 if job['name'] == name: 91 self.schedulers.remove(job) 92 break 93 94 def update_job(self, job, update_mask): 95 """Simulate update jobs.""" 96 for existing_job in self.schedulers: 97 if existing_job == job: 98 job['schedule'] = update_mask['schedule'] 99 100 101class TestDataSync(unittest.TestCase): 102 """Unit tests for sync.""" 103 104 @classmethod 105 def setUpClass(cls): 106 cls.ds_emulator = test_utils.start_datastore_emulator() 107 test_utils.wait_for_emulator_ready(cls.ds_emulator, 'datastore', 108 test_utils.DATASTORE_READY_INDICATOR) 109 test_utils.set_gcp_environment() 110 111 def setUp(self): 112 test_utils.reset_ds_emulator() 113 114 def test_sync_projects_update(self): 115 """Testing sync_projects() updating a schedule.""" 116 cloud_scheduler_client = CloudSchedulerClient() 117 118 with ndb.Client().context(): 119 Project(name='test1', 120 schedule='0 8 * * *', 121 project_yaml_contents='', 122 dockerfile_contents='').put() 123 Project(name='test2', 124 schedule='0 9 * * *', 125 project_yaml_contents='', 126 dockerfile_contents='').put() 127 128 projects = { 129 'test1': ProjectMetadata('0 8 * * *', '', ''), 130 'test2': ProjectMetadata('0 7 * * *', '', '') 131 } 132 sync_projects(cloud_scheduler_client, projects) 133 134 projects_query = Project.query() 135 self.assertEqual({ 136 'test1': '0 8 * * *', 137 'test2': '0 7 * * *' 138 }, {project.name: project.schedule for project in projects_query}) 139 140 def test_sync_projects_create(self): 141 """"Testing sync_projects() creating new schedule.""" 142 cloud_scheduler_client = CloudSchedulerClient() 143 144 with ndb.Client().context(): 145 Project(name='test1', 146 schedule='0 8 * * *', 147 project_yaml_contents='', 148 dockerfile_contents='').put() 149 150 projects = { 151 'test1': ProjectMetadata('0 8 * * *', '', ''), 152 'test2': ProjectMetadata('0 7 * * *', '', '') 153 } 154 sync_projects(cloud_scheduler_client, projects) 155 156 projects_query = Project.query() 157 self.assertEqual({ 158 'test1': '0 8 * * *', 159 'test2': '0 7 * * *' 160 }, {project.name: project.schedule for project in projects_query}) 161 162 self.assertCountEqual([ 163 { 164 'name': 'projects/test-project/location/us-central1/jobs/' 165 'test2-scheduler-fuzzing', 166 'pubsub_target': { 167 'topic_name': 'projects/test-project/topics/request-build', 168 'data': b'test2' 169 }, 170 'schedule': '0 7 * * *' 171 }, 172 { 173 'name': 'projects/test-project/location/us-central1/jobs/' 174 'test2-scheduler-coverage', 175 'pubsub_target': { 176 'topic_name': 177 'projects/test-project/topics/request-coverage-build', 178 'data': 179 b'test2' 180 }, 181 'schedule': '0 6 * * *' 182 }, 183 ], cloud_scheduler_client.schedulers) 184 185 def test_sync_projects_delete(self): 186 """Testing sync_projects() deleting.""" 187 cloud_scheduler_client = CloudSchedulerClient() 188 189 with ndb.Client().context(): 190 Project(name='test1', 191 schedule='0 8 * * *', 192 project_yaml_contents='', 193 dockerfile_contents='').put() 194 Project(name='test2', 195 schedule='0 9 * * *', 196 project_yaml_contents='', 197 dockerfile_contents='').put() 198 199 projects = {'test1': ProjectMetadata('0 8 * * *', '', '')} 200 sync_projects(cloud_scheduler_client, projects) 201 202 projects_query = Project.query() 203 self.assertEqual( 204 {'test1': '0 8 * * *'}, 205 {project.name: project.schedule for project in projects_query}) 206 207 def test_get_projects_yaml(self): 208 """Testing get_projects() yaml get_schedule().""" 209 210 repo = Repository('oss-fuzz', 'dir', 'projects', [ 211 Repository('test0', 'dir', 'projects/test0', [ 212 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 213 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 214 ]), 215 Repository('test1', 'dir', 'projects/test1', [ 216 Repository('Dockerfile', 'file', 'projects/test1/Dockerfile'), 217 Repository('project.yaml', 'file', 'projects/test1/project.yaml') 218 ]) 219 ]) 220 repo.contents[0].contents[1].set_yaml_contents(b'builds_per_day: 2') 221 repo.contents[1].contents[1].set_yaml_contents(b'builds_per_day: 3') 222 223 self.assertEqual( 224 get_projects(repo), { 225 'test0': 226 ProjectMetadata('0 6,18 * * *', 'builds_per_day: 2', 227 'name: test'), 228 'test1': 229 ProjectMetadata('0 6,14,22 * * *', 'builds_per_day: 3', 230 'name: test') 231 }) 232 233 def test_get_projects_no_docker_file(self): 234 """Testing get_projects() with missing dockerfile""" 235 236 repo = Repository('oss-fuzz', 'dir', 'projects', [ 237 Repository('test0', 'dir', 'projects/test0', [ 238 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 239 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 240 ]), 241 Repository('test1', 'dir', 'projects/test1') 242 ]) 243 244 self.assertEqual( 245 get_projects(repo), 246 {'test0': ProjectMetadata('0 6 * * *', 'name: test', 'name: test')}) 247 248 def test_get_projects_invalid_project_name(self): 249 """Testing get_projects() with invalid project name""" 250 251 repo = Repository('oss-fuzz', 'dir', 'projects', [ 252 Repository('test0', 'dir', 'projects/test0', [ 253 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 254 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 255 ]), 256 Repository('test1@', 'dir', 'projects/test1', [ 257 Repository('Dockerfile', 'file', 'projects/test1/Dockerfile'), 258 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 259 ]) 260 ]) 261 262 self.assertEqual( 263 get_projects(repo), 264 {'test0': ProjectMetadata('0 6 * * *', 'name: test', 'name: test')}) 265 266 def test_get_projects_non_directory_type_project(self): 267 """Testing get_projects() when a file in projects/ is not of type 'dir'.""" 268 269 repo = Repository('oss-fuzz', 'dir', 'projects', [ 270 Repository('test0', 'dir', 'projects/test0', [ 271 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 272 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 273 ]), 274 Repository('test1', 'file', 'projects/test1') 275 ]) 276 277 self.assertEqual( 278 get_projects(repo), 279 {'test0': ProjectMetadata('0 6 * * *', 'name: test', 'name: test')}) 280 281 def test_invalid_yaml_format(self): 282 """Testing invalid yaml schedule parameter argument.""" 283 284 repo = Repository('oss-fuzz', 'dir', 'projects', [ 285 Repository('test0', 'dir', 'projects/test0', [ 286 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 287 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 288 ]) 289 ]) 290 repo.contents[0].contents[1].set_yaml_contents( 291 b'builds_per_day: some-string') 292 293 self.assertEqual(get_projects(repo), {}) 294 295 def test_yaml_out_of_range(self): 296 """Testing invalid yaml schedule parameter argument.""" 297 298 repo = Repository('oss-fuzz', 'dir', 'projects', [ 299 Repository('test0', 'dir', 'projects/test0', [ 300 Repository('Dockerfile', 'file', 'projects/test0/Dockerfile'), 301 Repository('project.yaml', 'file', 'projects/test0/project.yaml') 302 ]) 303 ]) 304 repo.contents[0].contents[1].set_yaml_contents(b'builds_per_day: 5') 305 306 self.assertEqual(get_projects(repo), {}) 307 308 def test_get_github_creds(self): 309 """Testing get_github_creds().""" 310 with ndb.Client().context(): 311 self.assertRaises(RuntimeError, get_github_creds) 312 313 @classmethod 314 def tearDownClass(cls): 315 test_utils.cleanup_emulator(cls.ds_emulator) 316 317 318if __name__ == '__main__': 319 unittest.main(exit=False) 320