1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.targetprep; 18 19 import com.android.annotations.VisibleForTesting; 20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.command.remote.DeviceDescriptor; 23 import com.android.tradefed.config.Option; 24 import com.android.tradefed.config.OptionClass; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.device.ITestDevice; 27 import com.android.tradefed.invoker.IInvocationContext; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 30 import com.android.tradefed.util.CommandResult; 31 import com.android.tradefed.util.CommandStatus; 32 import com.android.tradefed.util.EnvUtil; 33 import com.android.tradefed.util.FileUtil; 34 import com.android.tradefed.util.IRunUtil; 35 import com.android.tradefed.util.RunUtil; 36 import com.android.tradefed.util.VtsPythonRunnerHelper; 37 import com.android.tradefed.util.VtsVendorConfigFileUtil; 38 39 import java.io.File; 40 import java.io.FileNotFoundException; 41 import java.io.IOException; 42 import java.util.Collection; 43 import java.util.LinkedHashSet; 44 import java.util.Map; 45 import java.util.NoSuchElementException; 46 import java.util.TreeMap; 47 import java.util.TreeSet; 48 49 /** 50 * Sets up a Python virtualenv on the host and installs packages. To activate it, the working 51 * directory is changed to the root of the virtualenv. 52 * 53 * This's a fork of PythonVirtualenvPreparer and is forked in order to simplify the change 54 * deployment process and reduce the deployment time, which are critical for VTS services. 55 * That means changes here will be upstreamed gradually. 56 */ 57 @OptionClass(alias = "python-venv") 58 public class VtsPythonVirtualenvPreparer implements IMultiTargetPreparer { 59 private static final String LOCAL_PYPI_PATH_ENV_VAR_NAME = "VTS_PYPI_PATH"; 60 private static final String LOCAL_PYPI_PATH_KEY = "pypi_packages_path"; 61 private static final int SECOND_IN_MSECS = 1000; 62 private static final int MINUTE_IN_MSECS = 60 * SECOND_IN_MSECS; 63 protected static final int PIP_RETRY = 3; 64 private static final int PIP_RETRY_WAIT = 3 * SECOND_IN_MSECS; 65 protected static final int PIP_INSTALL_DELAY = SECOND_IN_MSECS; 66 public static final String VIRTUAL_ENV_V3 = "VIRTUAL_ENV_V3"; 67 public static final String VIRTUAL_ENV = "VIRTUAL_ENV"; 68 69 @Option(name = "venv-dir", description = "path of an existing virtualenv to use") 70 protected File mVenvDir = null; 71 72 @Option(name = "requirements-file", description = "pip-formatted requirements file") 73 private File mRequirementsFile = null; 74 75 @Option(name = "script-file", description = "scripts which need to be executed in advance") 76 private Collection<String> mScriptFiles = new TreeSet<>(); 77 78 @Option(name = "dep-module", description = "modules which need to be installed by pip") 79 protected Collection<String> mDepModules = new LinkedHashSet<>(); 80 81 @Option(name = "no-dep-module", description = "modules which should not be installed by pip") 82 private Collection<String> mNoDepModules = new TreeSet<>(); 83 84 @Option(name = "reuse", 85 description = "Reuse an exising virtualenv path if exists in " 86 + "temp directory. When this option is enabled, virtualenv directory used or " 87 + "created by this preparer will not be deleted after tests complete.") 88 protected boolean mReuse = true; 89 90 @Option(name = "python-version", 91 description = "The version of a Python interpreter to use." 92 + "Currently, only major version number is fully supported." 93 + "Example: \"2\", or \"3\".") 94 private String mPythonVersion = "2"; 95 96 @Option(name = "virtual-env-intallation-wait-time", 97 isTimeVal = true, 98 description = "The maximum wait time for virtual env installation.") 99 private long mVirtualEnvInstallationWaitTime = 600000L; 100 101 private IBuildInfo mBuildInfo = null; 102 private DeviceDescriptor mDescriptor = null; 103 private IRunUtil mRunUtil = new RunUtil(); 104 105 String mLocalPypiPath = null; 106 String mPipPath = null; 107 108 // A map of initially installed pip modules and versions. Newly installed modules are not 109 // currently added automatically. 110 private Map<String, String> mPipInstallList = null; 111 112 /** 113 * {@inheritDoc} 114 */ 115 @Override setUp(IInvocationContext context)116 public void setUp(IInvocationContext context) 117 throws TargetSetupError, BuildError, DeviceNotAvailableException { 118 mBuildInfo = context.getBuildInfos().get(0); 119 ITestDevice device = context.getDevices().get(0); 120 mDescriptor = device.getDeviceDescriptor(); 121 // Ensure the method is locked even across instances 122 synchronized (VtsPythonVirtualenvPreparer.class) { 123 // Get virtual-env if existing 124 if (mVenvDir == null) { 125 mVenvDir = checkTestPlanLevelVirtualenv(mBuildInfo); 126 if (mVenvDir == null) { 127 mVenvDir = createVirtualEnvCache(mBuildInfo); 128 } 129 } 130 if (new File(mVenvDir, "complete").exists()) { 131 VtsPythonRunnerHelper.activateVirtualenv(getRunUtil(), mVenvDir.getAbsolutePath()); 132 } else { 133 // If cache is not good. 134 CLog.d("Preparing python dependencies..."); 135 136 if (!createVirtualenv(mVenvDir)) { 137 throw new TargetSetupError("Failed to create the virtual-env", mDescriptor); 138 } 139 CLog.d("Python virtualenv path is: " + mVenvDir); 140 VtsPythonRunnerHelper.activateVirtualenv(getRunUtil(), mVenvDir.getAbsolutePath()); 141 try { 142 new File(mVenvDir, "complete").createNewFile(); 143 } catch (IOException e) { 144 throw new TargetSetupError( 145 "Failed to mark virtualenv complete.", e, mDescriptor); 146 } 147 } 148 // Setup the dependencies no matter what. 149 setLocalPypiPath(); 150 installDeps(); 151 // Set the built virtual-env in the build info. 152 CLog.d("Python virtualenv path is: " + mVenvDir); 153 addPathToBuild(mBuildInfo, mVenvDir); 154 } 155 } 156 157 /** 158 * {@inheritDoc} 159 */ 160 @Override tearDown(IInvocationContext context, Throwable e)161 public void tearDown(IInvocationContext context, Throwable e) 162 throws DeviceNotAvailableException { 163 mVenvDir = null; 164 } 165 166 /** 167 * This method sets mLocalPypiPath, the local PyPI package directory to 168 * install python packages from in the installDeps method. 169 */ setLocalPypiPath()170 protected void setLocalPypiPath() { 171 VtsVendorConfigFileUtil configReader = new VtsVendorConfigFileUtil(); 172 if (configReader.LoadVendorConfig(mBuildInfo)) { 173 // First try to load local PyPI directory path from vendor config file 174 try { 175 String pypiPath = configReader.GetVendorConfigVariable(LOCAL_PYPI_PATH_KEY); 176 if (pypiPath.length() > 0 && dirExistsAndHaveReadAccess(pypiPath)) { 177 mLocalPypiPath = pypiPath; 178 CLog.d(String.format("Loaded %s: %s", LOCAL_PYPI_PATH_KEY, mLocalPypiPath)); 179 } 180 } catch (NoSuchElementException e) { 181 /* continue */ 182 } 183 } 184 185 // If loading path from vendor config file is unsuccessful, 186 // check local pypi path defined by LOCAL_PYPI_PATH_ENV_VAR_NAME 187 if (mLocalPypiPath == null) { 188 CLog.d("Checking whether local pypi packages directory exists"); 189 String pypiPath = System.getenv(LOCAL_PYPI_PATH_ENV_VAR_NAME); 190 if (pypiPath == null) { 191 CLog.d("Local pypi packages directory not specified by env var %s", 192 LOCAL_PYPI_PATH_ENV_VAR_NAME); 193 } else if (dirExistsAndHaveReadAccess(pypiPath)) { 194 mLocalPypiPath = pypiPath; 195 CLog.d("Set local pypi packages directory to %s", pypiPath); 196 } 197 } 198 199 if (mLocalPypiPath == null) { 200 CLog.d("Failed to set local pypi packages path. Therefore internet connection to " 201 + "https://pypi.python.org/simple/ must be available to run VTS tests."); 202 } 203 } 204 205 /** 206 * This method returns whether the given path is a dir that exists and the user has read access. 207 */ dirExistsAndHaveReadAccess(String path)208 private boolean dirExistsAndHaveReadAccess(String path) { 209 File pathDir = new File(path); 210 if (!pathDir.exists() || !pathDir.isDirectory()) { 211 CLog.d("Directory %s does not exist.", pathDir); 212 return false; 213 } 214 215 if (!EnvUtil.isOnWindows()) { 216 CommandResult c = getRunUtil().runTimedCmd(MINUTE_IN_MSECS, "ls", path); 217 if (c.getStatus() != CommandStatus.SUCCESS) { 218 CLog.d(String.format("Failed to read dir: %s. Result %s. stdout: %s, stderr: %s", 219 path, c.getStatus(), c.getStdout(), c.getStderr())); 220 return false; 221 } 222 return true; 223 } else { 224 try { 225 String[] pathDirList = pathDir.list(); 226 if (pathDirList == null) { 227 CLog.d("Failed to read dir: %s. Please check access permission.", pathDir); 228 return false; 229 } 230 } catch (SecurityException e) { 231 CLog.d(String.format( 232 "Failed to read dir %s with SecurityException %s", pathDir, e)); 233 return false; 234 } 235 return true; 236 } 237 } 238 239 /** 240 * Installs all python pip module dependencies specified in options. 241 * @throws TargetSetupError if failed 242 */ installDeps()243 protected void installDeps() throws TargetSetupError { 244 boolean hasDependencies = false; 245 if (!mScriptFiles.isEmpty()) { 246 for (String scriptFile : mScriptFiles) { 247 CLog.d("Attempting to execute a script, %s", scriptFile); 248 CommandResult c = getRunUtil().runTimedCmd(5 * MINUTE_IN_MSECS, scriptFile); 249 if (c.getStatus() != CommandStatus.SUCCESS) { 250 CLog.e("Executing script %s failed", scriptFile); 251 throw new TargetSetupError("Failed to source a script", mDescriptor); 252 } 253 } 254 } 255 256 if (mRequirementsFile != null) { 257 hasDependencies = true; 258 boolean success = false; 259 260 long retry_interval = PIP_RETRY_WAIT; 261 for (int try_count = 0; try_count < PIP_RETRY + 1; try_count++) { 262 if (try_count > 0) { 263 getRunUtil().sleep(retry_interval); 264 retry_interval *= 3; 265 } 266 267 if (installPipRequirementFile(mRequirementsFile)) { 268 success = true; 269 break; 270 } 271 } 272 273 if (!success) { 274 throw new TargetSetupError( 275 "Failed to install pip requirement file " + mRequirementsFile, mDescriptor); 276 } 277 } 278 279 if (!mDepModules.isEmpty()) { 280 for (String dep : mDepModules) { 281 hasDependencies = true; 282 283 if (mNoDepModules.contains(dep) || isPipModuleInstalled(dep)) { 284 continue; 285 } 286 287 boolean success = installPipModuleLocally(dep); 288 289 long retry_interval = PIP_RETRY_WAIT; 290 for (int retry_count = 0; retry_count < PIP_RETRY + 1; retry_count++) { 291 if (retry_count > 0) { 292 getRunUtil().sleep(retry_interval); 293 retry_interval *= 3; 294 } 295 296 if (success || (!success && installPipModule(dep))) { 297 success = true; 298 getRunUtil().sleep(PIP_INSTALL_DELAY); 299 break; 300 } 301 } 302 303 if (!success) { 304 throw new TargetSetupError("Failed to install pip module " + dep, mDescriptor); 305 } 306 } 307 } 308 if (!hasDependencies) { 309 CLog.d("No dependencies to install"); 310 } 311 } 312 313 /** 314 * Installs a pip requirement file from Internet. 315 * @param req pip module requirement file object 316 * @return true if success. False otherwise 317 */ installPipRequirementFile(File req)318 private boolean installPipRequirementFile(File req) { 319 CommandResult result = getRunUtil().runTimedCmd(10 * MINUTE_IN_MSECS, getPipPath(), 320 "install", "-r", mRequirementsFile.getAbsolutePath()); 321 CLog.d(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 322 result.getStdout(), result.getStderr())); 323 324 return result.getStatus() == CommandStatus.SUCCESS; 325 } 326 327 /** 328 * Installs a pip module from local directory. 329 * @param name of the module 330 * @return true if the module is successfully installed; false otherwise. 331 */ installPipModuleLocally(String name)332 private boolean installPipModuleLocally(String name) { 333 if (mLocalPypiPath == null) { 334 return false; 335 } 336 CLog.d("Attempting installation of %s from local directory", name); 337 CommandResult result = getRunUtil().runTimedCmd(5 * MINUTE_IN_MSECS, getPipPath(), 338 "install", name, "--no-index", "--find-links=" + mLocalPypiPath); 339 CLog.d(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 340 result.getStdout(), result.getStderr())); 341 342 return result.getStatus() == CommandStatus.SUCCESS; 343 } 344 345 /** 346 * Install a pip module from Internet 347 * @param name of the module 348 * @return true if success. False otherwise 349 */ installPipModule(String name)350 private boolean installPipModule(String name) { 351 CLog.d("Attempting installation of %s from PyPI", name); 352 CommandResult result = 353 getRunUtil().runTimedCmd(5 * MINUTE_IN_MSECS, getPipPath(), "install", name); 354 CLog.d("Result %s. stdout: %s, stderr: %s", result.getStatus(), result.getStdout(), 355 result.getStderr()); 356 if (result.getStatus() != CommandStatus.SUCCESS) { 357 CLog.e("Installing %s from PyPI failed.", name); 358 CLog.d("Attempting to upgrade %s", name); 359 result = getRunUtil().runTimedCmd( 360 5 * MINUTE_IN_MSECS, getPipPath(), "install", "--upgrade", name); 361 362 CLog.d(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 363 result.getStdout(), result.getStderr())); 364 } 365 366 return result.getStatus() == CommandStatus.SUCCESS; 367 } 368 369 /** 370 * This method returns absolute pip path in virtualenv. 371 * 372 * This method is needed because although PATH is set in IRunUtil, IRunUtil will still 373 * use pip from system path. 374 * 375 * @return absolute pip path in virtualenv. null if virtualenv not available. 376 */ getPipPath()377 public String getPipPath() { 378 if (mPipPath != null) { 379 return mPipPath; 380 } 381 382 String virtualenvPath = mVenvDir.getAbsolutePath(); 383 if (virtualenvPath == null) { 384 return null; 385 } 386 mPipPath = new File(VtsPythonRunnerHelper.getPythonBinDir(virtualenvPath), "pip") 387 .getAbsolutePath(); 388 return mPipPath; 389 } 390 391 /** 392 * Get the major python version from option. 393 * 394 * Currently, only 2 and 3 are supported. 395 * 396 * @return major version number 397 * @throws TargetSetupError 398 */ getConfiguredPythonVersionMajor()399 protected int getConfiguredPythonVersionMajor() throws TargetSetupError { 400 if (mPythonVersion.startsWith("3.") || mPythonVersion.equals("3")) { 401 return 3; 402 } else if (mPythonVersion.startsWith("2.") || mPythonVersion.equals("2")) { 403 return 2; 404 } else { 405 throw new TargetSetupError("Unsupported python version " + mPythonVersion, mDescriptor); 406 } 407 } 408 409 /** 410 * Add PYTHONPATH and VIRTUAL_ENV_PATH to BuildInfo. 411 * @param buildInfo 412 * @throws TargetSetupError 413 */ addPathToBuild(IBuildInfo buildInfo, File virtualEnvDir)414 private void addPathToBuild(IBuildInfo buildInfo, File virtualEnvDir) throws TargetSetupError { 415 String target = null; 416 switch (getConfiguredPythonVersionMajor()) { 417 case 2: 418 target = VtsPythonVirtualenvPreparer.VIRTUAL_ENV; 419 break; 420 case 3: 421 target = VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3; 422 break; 423 } 424 425 if (!buildInfo.getBuildAttributes().containsKey(target)) { 426 buildInfo.addBuildAttribute(target, virtualEnvDir.getAbsolutePath()); 427 } 428 } 429 430 /** 431 * Completes the creation of virtualenv. 432 * @return true if the directory is successfully prepared as virtualenv; false otherwise 433 */ createVirtualenv(File virtualEnvDir)434 protected boolean createVirtualenv(File virtualEnvDir) { 435 CLog.d("Creating virtualenv at " + virtualEnvDir); 436 437 String[] cmd = new String[] { 438 "virtualenv", "-p", "python" + mPythonVersion, virtualEnvDir.getAbsolutePath()}; 439 440 long waitRetryCreate = 5 * SECOND_IN_MSECS; 441 442 for (int try_count = 0; try_count < PIP_RETRY + 1; try_count++) { 443 if (try_count > 0) { 444 getRunUtil().sleep(waitRetryCreate); 445 } 446 CommandResult c = getRunUtil().runTimedCmd(mVirtualEnvInstallationWaitTime, cmd); 447 448 if (!CommandStatus.SUCCESS.equals(c.getStatus())) { 449 String message_lower = (c.getStdout() + c.getStderr()).toLowerCase(); 450 if (message_lower.contains("errno 17") // File exists 451 || message_lower.contains("errno 26") 452 || message_lower.contains("text file busy")) { 453 // Race condition, retry. 454 CLog.e("detected the virtualenv path is being created by other process."); 455 } else { 456 // Other error, abort. 457 CLog.e("Exit code: %s, stdout: %s, stderr: %s", c.getStatus(), c.getStdout(), 458 c.getStderr()); 459 break; 460 } 461 } else { 462 CLog.d("Successfully created virtualenv at " + virtualEnvDir); 463 return true; 464 } 465 } 466 467 return false; 468 } 469 createVirtualEnvCache(IBuildInfo buildInfo)470 private File createVirtualEnvCache(IBuildInfo buildInfo) throws TargetSetupError { 471 File workingDir = null; 472 File virtualEnvDir = null; 473 if (mReuse) { 474 try { 475 workingDir = new CompatibilityBuildHelper(buildInfo).getDir(); 476 } catch (FileNotFoundException e) { 477 workingDir = new File(System.getProperty("java.io.tmpdir")); 478 } 479 virtualEnvDir = new File(workingDir, "vts-virtualenv-" + mPythonVersion); 480 if (virtualEnvDir.exists()) { 481 // Use the cache 482 return virtualEnvDir; 483 } 484 // Create it first 485 virtualEnvDir.mkdirs(); 486 } else { 487 try { 488 virtualEnvDir = FileUtil.createTempDir("vts-virtualenv-" + mPythonVersion + "-" 489 + normalizeName(buildInfo.getTestTag()) + "_"); 490 } catch (IOException e) { 491 throw new TargetSetupError( 492 "Failed to create a directory for the virtual env.", e, mDescriptor); 493 } 494 } 495 return virtualEnvDir; 496 } 497 498 /** 499 * Checks whether a test plan-wise common virtualenv directory can be used. 500 * @param buildInfo 501 * @return true if a test plan-wise virtuanenv directory exists; false otherwise 502 * @throws TargetSetupError 503 */ checkTestPlanLevelVirtualenv(IBuildInfo buildInfo)504 protected File checkTestPlanLevelVirtualenv(IBuildInfo buildInfo) throws TargetSetupError { 505 String venvDir = null; 506 switch (getConfiguredPythonVersionMajor()) { 507 case 2: 508 venvDir = 509 buildInfo.getBuildAttributes().get(VtsPythonVirtualenvPreparer.VIRTUAL_ENV); 510 break; 511 case 3: 512 venvDir = buildInfo.getBuildAttributes().get( 513 VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3); 514 break; 515 } 516 517 if (venvDir != null && new File(venvDir).exists()) { 518 return new File(venvDir); 519 } 520 return null; 521 } 522 addDepModule(String module)523 protected void addDepModule(String module) { 524 mDepModules.add(module); 525 } 526 setRequirementsFile(File f)527 protected void setRequirementsFile(File f) { 528 mRequirementsFile = f; 529 } 530 531 /** 532 * Get an instance of {@link IRunUtil}. 533 */ 534 @VisibleForTesting getRunUtil()535 protected IRunUtil getRunUtil() { 536 if (mRunUtil == null) { 537 mRunUtil = new RunUtil(); 538 } 539 return mRunUtil; 540 } 541 542 /** 543 * Locally checks whether a pip module is installed. 544 * 545 * This read the installed module list from command "pip list" and check whether the 546 * module in requirement string is installed and its version satisfied. 547 * 548 * Note: This method is only a help method for speed optimization purpose. 549 * It does not check dependencies of the module. 550 * It replace dots "." in module name with dash "-". 551 * If the "pip list" command failed, it will return false and will not throw exception 552 * It can also only accept one ">=" version requirement string. 553 * If this method returns false, the requirement should still be checked using pip itself. 554 * 555 * @param requirement such as "numpy", "pip>=9" 556 * @return True if module is installed locally with correct version. False otherwise 557 */ isPipModuleInstalled(String requirement)558 private boolean isPipModuleInstalled(String requirement) { 559 if (mPipInstallList == null) { 560 mPipInstallList = getInstalledPipModules(); 561 if (mPipInstallList == null) { 562 CLog.e("Failed to read local pip install list."); 563 return false; 564 } 565 } 566 567 String name; 568 String version = null; 569 if (requirement.contains(">=")) { 570 String[] tokens = requirement.split(">="); 571 if (tokens.length != 2) { 572 return false; 573 } 574 name = tokens[0]; 575 version = tokens[1]; 576 } else if (requirement.contains("=") || requirement.contains("<") 577 || requirement.contains(">")) { 578 return false; 579 } else { 580 name = requirement; 581 } 582 583 name = name.replaceAll("\\.", "-"); 584 585 if (!mPipInstallList.containsKey(name)) { 586 return false; 587 } 588 589 // TODO: support other comparison and multiple condition if there's a use case. 590 if (version != null && !isVersionGreaterEqual(mPipInstallList.get(name), version)) { 591 return false; 592 } 593 594 return true; 595 } 596 597 /** 598 * Compares whether version string 1 is greater or equal to version string 2 599 * @param version1 600 * @param version2 601 * @return True if the value of version1 >= version2 602 */ isVersionGreaterEqual(String version1, String version2)603 private static boolean isVersionGreaterEqual(String version1, String version2) { 604 version1 = version1.replaceAll("[^0-9.]+", ""); 605 version2 = version2.replaceAll("[^0-9.]+", ""); 606 607 String[] tokens1 = version1.split("\\."); 608 String[] tokens2 = version2.split("\\."); 609 610 int length = Math.max(tokens1.length, tokens2.length); 611 for (int i = 0; i < length; i++) { 612 try { 613 int token1 = i < tokens1.length ? Integer.parseInt(tokens1[i]) : 0; 614 int token2 = i < tokens2.length ? Integer.parseInt(tokens2[i]) : 0; 615 if (token1 < token2) { 616 return false; 617 } 618 } catch (NumberFormatException e) { 619 CLog.e("failed to compare pip module version: %s and %s", version1, version2); 620 return false; 621 } 622 } 623 624 return true; 625 } 626 627 /** 628 * Gets map of installed pip packages and their versions. 629 * @return installed pip packages 630 */ 631 private Map<String, String> getInstalledPipModules() { 632 CommandResult res = getRunUtil().runTimedCmd(30 * SECOND_IN_MSECS, getPipPath(), "list"); 633 if (res.getStatus() != CommandStatus.SUCCESS) { 634 CLog.e(String.format("Failed to read pip installed list: " 635 + "Result %s. stdout: %s, stderr: %s", 636 res.getStatus(), res.getStdout(), res.getStderr())); 637 return null; 638 } 639 String raw = res.getStdout(); 640 String[] lines = raw.split("\\r?\\n"); 641 642 TreeMap<String, String> pipInstallList = new TreeMap<>(); 643 644 for (String line : lines) { 645 line = line.trim(); 646 if (line.length() == 0 || line.startsWith("Package ") || line.startsWith("-")) { 647 continue; 648 } 649 String[] tokens = line.split("\\s+"); 650 if (tokens.length != 2) { 651 CLog.e("Error parsing pip installed package list. Line text: " + line); 652 continue; 653 } 654 pipInstallList.put(tokens[0], tokens[1]); 655 } 656 657 return pipInstallList; 658 } 659 660 /** 661 * Replacing characters in a string to make it a valid file name. 662 * 663 * The current method is to replace any non-word character with '_' except '.' and '-'. 664 * 665 * @param name the potential name of a file to normalize. 666 * Do not use path here as path delimitor will be replaced 667 * @return normalized file name 668 */ 669 private String normalizeName(String name) { 670 return name.replaceAll("[^\\w.-]", "_"); 671 } 672 } 673