1"""setuptools.command.bdist_egg
2
3Build .egg distributions"""
4
5from distutils.errors import DistutilsSetupError
6from distutils.dir_util import remove_tree, mkpath
7from distutils import log
8from types import CodeType
9import sys
10import os
11import re
12import textwrap
13import marshal
14
15from setuptools.extern import six
16
17from pkg_resources import get_build_platform, Distribution, ensure_directory
18from pkg_resources import EntryPoint
19from setuptools.extension import Library
20from setuptools import Command
21
22try:
23    # Python 2.7 or >=3.2
24    from sysconfig import get_path, get_python_version
25
26    def _get_purelib():
27        return get_path("purelib")
28except ImportError:
29    from distutils.sysconfig import get_python_lib, get_python_version
30
31    def _get_purelib():
32        return get_python_lib(False)
33
34
35def strip_module(filename):
36    if '.' in filename:
37        filename = os.path.splitext(filename)[0]
38    if filename.endswith('module'):
39        filename = filename[:-6]
40    return filename
41
42
43def sorted_walk(dir):
44    """Do os.walk in a reproducible way,
45    independent of indeterministic filesystem readdir order
46    """
47    for base, dirs, files in os.walk(dir):
48        dirs.sort()
49        files.sort()
50        yield base, dirs, files
51
52
53def write_stub(resource, pyfile):
54    _stub_template = textwrap.dedent("""
55        def __bootstrap__():
56            global __bootstrap__, __loader__, __file__
57            import sys, pkg_resources, imp
58            __file__ = pkg_resources.resource_filename(__name__, %r)
59            __loader__ = None; del __bootstrap__, __loader__
60            imp.load_dynamic(__name__,__file__)
61        __bootstrap__()
62        """).lstrip()
63    with open(pyfile, 'w') as f:
64        f.write(_stub_template % resource)
65
66
67class bdist_egg(Command):
68    description = "create an \"egg\" distribution"
69
70    user_options = [
71        ('bdist-dir=', 'b',
72         "temporary directory for creating the distribution"),
73        ('plat-name=', 'p', "platform name to embed in generated filenames "
74                            "(default: %s)" % get_build_platform()),
75        ('exclude-source-files', None,
76         "remove all .py files from the generated egg"),
77        ('keep-temp', 'k',
78         "keep the pseudo-installation tree around after " +
79         "creating the distribution archive"),
80        ('dist-dir=', 'd',
81         "directory to put final built distributions in"),
82        ('skip-build', None,
83         "skip rebuilding everything (for testing/debugging)"),
84    ]
85
86    boolean_options = [
87        'keep-temp', 'skip-build', 'exclude-source-files'
88    ]
89
90    def initialize_options(self):
91        self.bdist_dir = None
92        self.plat_name = None
93        self.keep_temp = 0
94        self.dist_dir = None
95        self.skip_build = 0
96        self.egg_output = None
97        self.exclude_source_files = None
98
99    def finalize_options(self):
100        ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info")
101        self.egg_info = ei_cmd.egg_info
102
103        if self.bdist_dir is None:
104            bdist_base = self.get_finalized_command('bdist').bdist_base
105            self.bdist_dir = os.path.join(bdist_base, 'egg')
106
107        if self.plat_name is None:
108            self.plat_name = get_build_platform()
109
110        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))
111
112        if self.egg_output is None:
113
114            # Compute filename of the output egg
115            basename = Distribution(
116                None, None, ei_cmd.egg_name, ei_cmd.egg_version,
117                get_python_version(),
118                self.distribution.has_ext_modules() and self.plat_name
119            ).egg_name()
120
121            self.egg_output = os.path.join(self.dist_dir, basename + '.egg')
122
123    def do_install_data(self):
124        # Hack for packages that install data to install's --install-lib
125        self.get_finalized_command('install').install_lib = self.bdist_dir
126
127        site_packages = os.path.normcase(os.path.realpath(_get_purelib()))
128        old, self.distribution.data_files = self.distribution.data_files, []
129
130        for item in old:
131            if isinstance(item, tuple) and len(item) == 2:
132                if os.path.isabs(item[0]):
133                    realpath = os.path.realpath(item[0])
134                    normalized = os.path.normcase(realpath)
135                    if normalized == site_packages or normalized.startswith(
136                        site_packages + os.sep
137                    ):
138                        item = realpath[len(site_packages) + 1:], item[1]
139                        # XXX else: raise ???
140            self.distribution.data_files.append(item)
141
142        try:
143            log.info("installing package data to %s", self.bdist_dir)
144            self.call_command('install_data', force=0, root=None)
145        finally:
146            self.distribution.data_files = old
147
148    def get_outputs(self):
149        return [self.egg_output]
150
151    def call_command(self, cmdname, **kw):
152        """Invoke reinitialized command `cmdname` with keyword args"""
153        for dirname in INSTALL_DIRECTORY_ATTRS:
154            kw.setdefault(dirname, self.bdist_dir)
155        kw.setdefault('skip_build', self.skip_build)
156        kw.setdefault('dry_run', self.dry_run)
157        cmd = self.reinitialize_command(cmdname, **kw)
158        self.run_command(cmdname)
159        return cmd
160
161    def run(self):
162        # Generate metadata first
163        self.run_command("egg_info")
164        # We run install_lib before install_data, because some data hacks
165        # pull their data path from the install_lib command.
166        log.info("installing library code to %s", self.bdist_dir)
167        instcmd = self.get_finalized_command('install')
168        old_root = instcmd.root
169        instcmd.root = None
170        if self.distribution.has_c_libraries() and not self.skip_build:
171            self.run_command('build_clib')
172        cmd = self.call_command('install_lib', warn_dir=0)
173        instcmd.root = old_root
174
175        all_outputs, ext_outputs = self.get_ext_outputs()
176        self.stubs = []
177        to_compile = []
178        for (p, ext_name) in enumerate(ext_outputs):
179            filename, ext = os.path.splitext(ext_name)
180            pyfile = os.path.join(self.bdist_dir, strip_module(filename) +
181                                  '.py')
182            self.stubs.append(pyfile)
183            log.info("creating stub loader for %s", ext_name)
184            if not self.dry_run:
185                write_stub(os.path.basename(ext_name), pyfile)
186            to_compile.append(pyfile)
187            ext_outputs[p] = ext_name.replace(os.sep, '/')
188
189        if to_compile:
190            cmd.byte_compile(to_compile)
191        if self.distribution.data_files:
192            self.do_install_data()
193
194        # Make the EGG-INFO directory
195        archive_root = self.bdist_dir
196        egg_info = os.path.join(archive_root, 'EGG-INFO')
197        self.mkpath(egg_info)
198        if self.distribution.scripts:
199            script_dir = os.path.join(egg_info, 'scripts')
200            log.info("installing scripts to %s", script_dir)
201            self.call_command('install_scripts', install_dir=script_dir,
202                              no_ep=1)
203
204        self.copy_metadata_to(egg_info)
205        native_libs = os.path.join(egg_info, "native_libs.txt")
206        if all_outputs:
207            log.info("writing %s", native_libs)
208            if not self.dry_run:
209                ensure_directory(native_libs)
210                libs_file = open(native_libs, 'wt')
211                libs_file.write('\n'.join(all_outputs))
212                libs_file.write('\n')
213                libs_file.close()
214        elif os.path.isfile(native_libs):
215            log.info("removing %s", native_libs)
216            if not self.dry_run:
217                os.unlink(native_libs)
218
219        write_safety_flag(
220            os.path.join(archive_root, 'EGG-INFO'), self.zip_safe()
221        )
222
223        if os.path.exists(os.path.join(self.egg_info, 'depends.txt')):
224            log.warn(
225                "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"
226                "Use the install_requires/extras_require setup() args instead."
227            )
228
229        if self.exclude_source_files:
230            self.zap_pyfiles()
231
232        # Make the archive
233        make_zipfile(self.egg_output, archive_root, verbose=self.verbose,
234                     dry_run=self.dry_run, mode=self.gen_header())
235        if not self.keep_temp:
236            remove_tree(self.bdist_dir, dry_run=self.dry_run)
237
238        # Add to 'Distribution.dist_files' so that the "upload" command works
239        getattr(self.distribution, 'dist_files', []).append(
240            ('bdist_egg', get_python_version(), self.egg_output))
241
242    def zap_pyfiles(self):
243        log.info("Removing .py files from temporary directory")
244        for base, dirs, files in walk_egg(self.bdist_dir):
245            for name in files:
246                path = os.path.join(base, name)
247
248                if name.endswith('.py'):
249                    log.debug("Deleting %s", path)
250                    os.unlink(path)
251
252                if base.endswith('__pycache__'):
253                    path_old = path
254
255                    pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc'
256                    m = re.match(pattern, name)
257                    path_new = os.path.join(
258                        base, os.pardir, m.group('name') + '.pyc')
259                    log.info(
260                        "Renaming file from [%s] to [%s]"
261                        % (path_old, path_new))
262                    try:
263                        os.remove(path_new)
264                    except OSError:
265                        pass
266                    os.rename(path_old, path_new)
267
268    def zip_safe(self):
269        safe = getattr(self.distribution, 'zip_safe', None)
270        if safe is not None:
271            return safe
272        log.warn("zip_safe flag not set; analyzing archive contents...")
273        return analyze_egg(self.bdist_dir, self.stubs)
274
275    def gen_header(self):
276        epm = EntryPoint.parse_map(self.distribution.entry_points or '')
277        ep = epm.get('setuptools.installation', {}).get('eggsecutable')
278        if ep is None:
279            return 'w'  # not an eggsecutable, do it the usual way.
280
281        if not ep.attrs or ep.extras:
282            raise DistutilsSetupError(
283                "eggsecutable entry point (%r) cannot have 'extras' "
284                "or refer to a module" % (ep,)
285            )
286
287        pyver = sys.version[:3]
288        pkg = ep.module_name
289        full = '.'.join(ep.attrs)
290        base = ep.attrs[0]
291        basename = os.path.basename(self.egg_output)
292
293        header = (
294            "#!/bin/sh\n"
295            'if [ `basename $0` = "%(basename)s" ]\n'
296            'then exec python%(pyver)s -c "'
297            "import sys, os; sys.path.insert(0, os.path.abspath('$0')); "
298            "from %(pkg)s import %(base)s; sys.exit(%(full)s())"
299            '" "$@"\n'
300            'else\n'
301            '  echo $0 is not the correct name for this egg file.\n'
302            '  echo Please rename it back to %(basename)s and try again.\n'
303            '  exec false\n'
304            'fi\n'
305        ) % locals()
306
307        if not self.dry_run:
308            mkpath(os.path.dirname(self.egg_output), dry_run=self.dry_run)
309            f = open(self.egg_output, 'w')
310            f.write(header)
311            f.close()
312        return 'a'
313
314    def copy_metadata_to(self, target_dir):
315        "Copy metadata (egg info) to the target_dir"
316        # normalize the path (so that a forward-slash in egg_info will
317        # match using startswith below)
318        norm_egg_info = os.path.normpath(self.egg_info)
319        prefix = os.path.join(norm_egg_info, '')
320        for path in self.ei_cmd.filelist.files:
321            if path.startswith(prefix):
322                target = os.path.join(target_dir, path[len(prefix):])
323                ensure_directory(target)
324                self.copy_file(path, target)
325
326    def get_ext_outputs(self):
327        """Get a list of relative paths to C extensions in the output distro"""
328
329        all_outputs = []
330        ext_outputs = []
331
332        paths = {self.bdist_dir: ''}
333        for base, dirs, files in sorted_walk(self.bdist_dir):
334            for filename in files:
335                if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS:
336                    all_outputs.append(paths[base] + filename)
337            for filename in dirs:
338                paths[os.path.join(base, filename)] = (paths[base] +
339                                                       filename + '/')
340
341        if self.distribution.has_ext_modules():
342            build_cmd = self.get_finalized_command('build_ext')
343            for ext in build_cmd.extensions:
344                if isinstance(ext, Library):
345                    continue
346                fullname = build_cmd.get_ext_fullname(ext.name)
347                filename = build_cmd.get_ext_filename(fullname)
348                if not os.path.basename(filename).startswith('dl-'):
349                    if os.path.exists(os.path.join(self.bdist_dir, filename)):
350                        ext_outputs.append(filename)
351
352        return all_outputs, ext_outputs
353
354
355NATIVE_EXTENSIONS = dict.fromkeys('.dll .so .dylib .pyd'.split())
356
357
358def walk_egg(egg_dir):
359    """Walk an unpacked egg's contents, skipping the metadata directory"""
360    walker = sorted_walk(egg_dir)
361    base, dirs, files = next(walker)
362    if 'EGG-INFO' in dirs:
363        dirs.remove('EGG-INFO')
364    yield base, dirs, files
365    for bdf in walker:
366        yield bdf
367
368
369def analyze_egg(egg_dir, stubs):
370    # check for existing flag in EGG-INFO
371    for flag, fn in safety_flags.items():
372        if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)):
373            return flag
374    if not can_scan():
375        return False
376    safe = True
377    for base, dirs, files in walk_egg(egg_dir):
378        for name in files:
379            if name.endswith('.py') or name.endswith('.pyw'):
380                continue
381            elif name.endswith('.pyc') or name.endswith('.pyo'):
382                # always scan, even if we already know we're not safe
383                safe = scan_module(egg_dir, base, name, stubs) and safe
384    return safe
385
386
387def write_safety_flag(egg_dir, safe):
388    # Write or remove zip safety flag file(s)
389    for flag, fn in safety_flags.items():
390        fn = os.path.join(egg_dir, fn)
391        if os.path.exists(fn):
392            if safe is None or bool(safe) != flag:
393                os.unlink(fn)
394        elif safe is not None and bool(safe) == flag:
395            f = open(fn, 'wt')
396            f.write('\n')
397            f.close()
398
399
400safety_flags = {
401    True: 'zip-safe',
402    False: 'not-zip-safe',
403}
404
405
406def scan_module(egg_dir, base, name, stubs):
407    """Check whether module possibly uses unsafe-for-zipfile stuff"""
408
409    filename = os.path.join(base, name)
410    if filename[:-1] in stubs:
411        return True  # Extension module
412    pkg = base[len(egg_dir) + 1:].replace(os.sep, '.')
413    module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0]
414    if sys.version_info < (3, 3):
415        skip = 8  # skip magic & date
416    elif sys.version_info < (3, 7):
417        skip = 12  # skip magic & date & file size
418    else:
419        skip = 16  # skip magic & reserved? & date & file size
420    f = open(filename, 'rb')
421    f.read(skip)
422    code = marshal.load(f)
423    f.close()
424    safe = True
425    symbols = dict.fromkeys(iter_symbols(code))
426    for bad in ['__file__', '__path__']:
427        if bad in symbols:
428            log.warn("%s: module references %s", module, bad)
429            safe = False
430    if 'inspect' in symbols:
431        for bad in [
432            'getsource', 'getabsfile', 'getsourcefile', 'getfile'
433            'getsourcelines', 'findsource', 'getcomments', 'getframeinfo',
434            'getinnerframes', 'getouterframes', 'stack', 'trace'
435        ]:
436            if bad in symbols:
437                log.warn("%s: module MAY be using inspect.%s", module, bad)
438                safe = False
439    return safe
440
441
442def iter_symbols(code):
443    """Yield names and strings used by `code` and its nested code objects"""
444    for name in code.co_names:
445        yield name
446    for const in code.co_consts:
447        if isinstance(const, six.string_types):
448            yield const
449        elif isinstance(const, CodeType):
450            for name in iter_symbols(const):
451                yield name
452
453
454def can_scan():
455    if not sys.platform.startswith('java') and sys.platform != 'cli':
456        # CPython, PyPy, etc.
457        return True
458    log.warn("Unable to analyze compiled code on this platform.")
459    log.warn("Please ask the author to include a 'zip_safe'"
460             " setting (either True or False) in the package's setup.py")
461
462
463# Attribute names of options for commands that might need to be convinced to
464# install to the egg build directory
465
466INSTALL_DIRECTORY_ATTRS = [
467    'install_lib', 'install_dir', 'install_data', 'install_base'
468]
469
470
471def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
472                 mode='w'):
473    """Create a zip file from all the files under 'base_dir'.  The output
474    zip file will be named 'base_dir' + ".zip".  Uses either the "zipfile"
475    Python module (if available) or the InfoZIP "zip" utility (if installed
476    and found on the default search path).  If neither tool is available,
477    raises DistutilsExecError.  Returns the name of the output zip file.
478    """
479    import zipfile
480
481    mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
482    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
483
484    def visit(z, dirname, names):
485        for name in names:
486            path = os.path.normpath(os.path.join(dirname, name))
487            if os.path.isfile(path):
488                p = path[len(base_dir) + 1:]
489                if not dry_run:
490                    z.write(path, p)
491                log.debug("adding '%s'", p)
492
493    compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
494    if not dry_run:
495        z = zipfile.ZipFile(zip_filename, mode, compression=compression)
496        for dirname, dirs, files in sorted_walk(base_dir):
497            visit(z, dirname, files)
498        z.close()
499    else:
500        for dirname, dirs, files in sorted_walk(base_dir):
501            visit(None, dirname, files)
502    return zip_filename
503