1"""
2Shared setup file for simple python packages. Uses a setup.cfg that
3is the same as the distutils2 project, unless noted otherwise.
4
5It exists for two reasons:
61) This makes it easier to reuse setup.py code between my own
7   projects
8
92) Easier migration to distutils2 when that catches on.
10
11Additional functionality:
12
13* Section metadata:
14    requires-test:  Same as 'tests_require' option for setuptools.
15
16"""
17
18import sys
19import os
20import re
21import platform
22from fnmatch import fnmatch
23import os
24import sys
25import time
26import tempfile
27import tarfile
28try:
29    import urllib.request as urllib
30except ImportError:
31    import urllib
32from distutils import log
33try:
34    from hashlib import md5
35
36except ImportError:
37    from md5 import md5
38
39if sys.version_info[0] == 2:
40    from ConfigParser import RawConfigParser, NoOptionError, NoSectionError
41else:
42    from configparser import RawConfigParser, NoOptionError, NoSectionError
43
44ROOTDIR = os.path.dirname(os.path.abspath(__file__))
45
46
47#
48#
49#
50# Parsing the setup.cfg and converting it to something that can be
51# used by setuptools.setup()
52#
53#
54#
55
56def eval_marker(value):
57    """
58    Evaluate an distutils2 environment marker.
59
60    This code is unsafe when used with hostile setup.cfg files,
61    but that's not a problem for our own files.
62    """
63    value = value.strip()
64
65    class M:
66        def __init__(self, **kwds):
67            for k, v in kwds.items():
68                setattr(self, k, v)
69
70    variables = {
71        'python_version': '%d.%d'%(sys.version_info[0], sys.version_info[1]),
72        'python_full_version': sys.version.split()[0],
73        'os': M(
74            name=os.name,
75        ),
76        'sys': M(
77            platform=sys.platform,
78        ),
79        'platform': M(
80            version=platform.version(),
81            machine=platform.machine(),
82        ),
83    }
84
85    return bool(eval(value, variables, variables))
86
87
88    return True
89
90def _opt_value(cfg, into, section, key, transform = None):
91    try:
92        v = cfg.get(section, key)
93        if transform != _as_lines and ';' in v:
94            v, marker = v.rsplit(';', 1)
95            if not eval_marker(marker):
96                return
97
98            v = v.strip()
99
100        if v:
101            if transform:
102                into[key] = transform(v.strip())
103            else:
104                into[key] = v.strip()
105
106    except (NoOptionError, NoSectionError):
107        pass
108
109def _as_bool(value):
110    if value.lower() in ('y', 'yes', 'on'):
111        return True
112    elif value.lower() in ('n', 'no', 'off'):
113        return False
114    elif value.isdigit():
115        return bool(int(value))
116    else:
117        raise ValueError(value)
118
119def _as_list(value):
120    return value.split()
121
122def _as_lines(value):
123    result = []
124    for v in value.splitlines():
125        if ';' in v:
126            v, marker = v.rsplit(';', 1)
127            if not eval_marker(marker):
128                continue
129
130            v = v.strip()
131            if v:
132                result.append(v)
133        else:
134            result.append(v)
135    return result
136
137def _map_requirement(value):
138    m = re.search(r'(\S+)\s*(?:\((.*)\))?', value)
139    name = m.group(1)
140    version = m.group(2)
141
142    if version is None:
143        return name
144
145    else:
146        mapped = []
147        for v in version.split(','):
148            v = v.strip()
149            if v[0].isdigit():
150                # Checks for a specific version prefix
151                m = v.rsplit('.', 1)
152                mapped.append('>=%s,<%s.%s'%(
153                    v, m[0], int(m[1])+1))
154
155            else:
156                mapped.append(v)
157        return '%s %s'%(name, ','.join(mapped),)
158
159def _as_requires(value):
160    requires = []
161    for req in value.splitlines():
162        if ';' in req:
163            req, marker = v.rsplit(';', 1)
164            if not eval_marker(marker):
165                continue
166            req = req.strip()
167
168        if not req:
169            continue
170        requires.append(_map_requirement(req))
171    return requires
172
173def parse_setup_cfg():
174    cfg = RawConfigParser()
175    r = cfg.read([os.path.join(ROOTDIR, 'setup.cfg')])
176    if len(r) != 1:
177        print("Cannot read 'setup.cfg'")
178        sys.exit(1)
179
180    metadata = dict(
181            name        = cfg.get('metadata', 'name'),
182            version     = cfg.get('metadata', 'version'),
183            description = cfg.get('metadata', 'description'),
184    )
185
186    _opt_value(cfg, metadata, 'metadata', 'license')
187    _opt_value(cfg, metadata, 'metadata', 'maintainer')
188    _opt_value(cfg, metadata, 'metadata', 'maintainer_email')
189    _opt_value(cfg, metadata, 'metadata', 'author')
190    _opt_value(cfg, metadata, 'metadata', 'author_email')
191    _opt_value(cfg, metadata, 'metadata', 'url')
192    _opt_value(cfg, metadata, 'metadata', 'download_url')
193    _opt_value(cfg, metadata, 'metadata', 'classifiers', _as_lines)
194    _opt_value(cfg, metadata, 'metadata', 'platforms', _as_list)
195    _opt_value(cfg, metadata, 'metadata', 'packages', _as_list)
196    _opt_value(cfg, metadata, 'metadata', 'keywords', _as_list)
197
198    try:
199        v = cfg.get('metadata', 'requires-dist')
200
201    except (NoOptionError, NoSectionError):
202        pass
203
204    else:
205        requires = _as_requires(v)
206        if requires:
207            metadata['install_requires'] = requires
208
209    try:
210        v = cfg.get('metadata', 'requires-test')
211
212    except (NoOptionError, NoSectionError):
213        pass
214
215    else:
216        requires = _as_requires(v)
217        if requires:
218            metadata['tests_require'] = requires
219
220
221    try:
222        v = cfg.get('metadata', 'long_description_file')
223    except (NoOptionError, NoSectionError):
224        pass
225
226    else:
227        parts = []
228        for nm in v.split():
229            fp = open(nm, 'rU')
230            parts.append(fp.read())
231            fp.close()
232
233        metadata['long_description'] = '\n\n'.join(parts)
234
235
236    try:
237        v = cfg.get('metadata', 'zip-safe')
238    except (NoOptionError, NoSectionError):
239        pass
240
241    else:
242        metadata['zip_safe'] = _as_bool(v)
243
244    try:
245        v = cfg.get('metadata', 'console_scripts')
246    except (NoOptionError, NoSectionError):
247        pass
248
249    else:
250        if 'entry_points' not in metadata:
251            metadata['entry_points'] = {}
252
253        metadata['entry_points']['console_scripts'] = v.splitlines()
254
255    if sys.version_info[:2] <= (2,6):
256        try:
257            metadata['tests_require'] += ", unittest2"
258        except KeyError:
259            metadata['tests_require'] = "unittest2"
260
261    return metadata
262
263
264#
265#
266#
267# Bootstrapping setuptools/distribute, based on
268# a heavily modified version of distribute_setup.py
269#
270#
271#
272
273
274SETUPTOOLS_PACKAGE='setuptools'
275
276
277try:
278    import subprocess
279
280    def _python_cmd(*args):
281        args = (sys.executable,) + args
282        return subprocess.call(args) == 0
283
284except ImportError:
285    def _python_cmd(*args):
286        args = (sys.executable,) + args
287        new_args = []
288        for a in args:
289            new_args.append(a.replace("'", "'\"'\"'"))
290        os.system(' '.join(new_args)) == 0
291
292
293try:
294    import json
295
296    def get_pypi_src_download(package):
297        url = 'https://pypi.python.org/pypi/%s/json'%(package,)
298        fp = urllib.urlopen(url)
299        try:
300            try:
301                data = fp.read()
302
303            finally:
304                fp.close()
305        except urllib.error:
306            raise RuntimeError("Cannot determine download link for %s"%(package,))
307
308        pkgdata = json.loads(data.decode('utf-8'))
309        if 'urls' not in pkgdata:
310            raise RuntimeError("Cannot determine download link for %s"%(package,))
311
312        for info in pkgdata['urls']:
313            if info['packagetype'] == 'sdist' and info['url'].endswith('tar.gz'):
314                return (info.get('md5_digest'), info['url'])
315
316        raise RuntimeError("Cannot determine downlink link for %s"%(package,))
317
318except ImportError:
319    # Python 2.5 compatibility, no JSON in stdlib but luckily JSON syntax is
320    # simular enough to Python's syntax to be able to abuse the Python compiler
321
322    import _ast as ast
323
324    def get_pypi_src_download(package):
325        url = 'https://pypi.python.org/pypi/%s/json'%(package,)
326        fp = urllib.urlopen(url)
327        try:
328            try:
329                data = fp.read()
330
331            finally:
332                fp.close()
333        except urllib.error:
334            raise RuntimeError("Cannot determine download link for %s"%(package,))
335
336
337        a = compile(data, '-', 'eval', ast.PyCF_ONLY_AST)
338        if not isinstance(a, ast.Expression):
339            raise RuntimeError("Cannot determine download link for %s"%(package,))
340
341        a = a.body
342        if not isinstance(a, ast.Dict):
343            raise RuntimeError("Cannot determine download link for %s"%(package,))
344
345        for k, v in zip(a.keys, a.values):
346            if not isinstance(k, ast.Str):
347                raise RuntimeError("Cannot determine download link for %s"%(package,))
348
349            k = k.s
350            if k == 'urls':
351                a = v
352                break
353        else:
354            raise RuntimeError("PyPI JSON for %s doesn't contain URLs section"%(package,))
355
356        if not isinstance(a, ast.List):
357            raise RuntimeError("Cannot determine download link for %s"%(package,))
358
359        for info in v.elts:
360            if not isinstance(info, ast.Dict):
361                raise RuntimeError("Cannot determine download link for %s"%(package,))
362            url = None
363            packagetype = None
364            chksum = None
365
366            for k, v in zip(info.keys, info.values):
367                if not isinstance(k, ast.Str):
368                    raise RuntimeError("Cannot determine download link for %s"%(package,))
369
370                if k.s == 'url':
371                    if not isinstance(v, ast.Str):
372                        raise RuntimeError("Cannot determine download link for %s"%(package,))
373                    url = v.s
374
375                elif k.s == 'packagetype':
376                    if not isinstance(v, ast.Str):
377                        raise RuntimeError("Cannot determine download link for %s"%(package,))
378                    packagetype = v.s
379
380                elif k.s == 'md5_digest':
381                    if not isinstance(v, ast.Str):
382                        raise RuntimeError("Cannot determine download link for %s"%(package,))
383                    chksum = v.s
384
385            if url is not None and packagetype == 'sdist' and url.endswith('.tar.gz'):
386                return (chksum, url)
387
388        raise RuntimeError("Cannot determine download link for %s"%(package,))
389
390def _build_egg(egg, tarball, to_dir):
391    # extracting the tarball
392    tmpdir = tempfile.mkdtemp()
393    log.warn('Extracting in %s', tmpdir)
394    old_wd = os.getcwd()
395    try:
396        os.chdir(tmpdir)
397        tar = tarfile.open(tarball)
398        _extractall(tar)
399        tar.close()
400
401        # going in the directory
402        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
403        os.chdir(subdir)
404        log.warn('Now working in %s', subdir)
405
406        # building an egg
407        log.warn('Building a %s egg in %s', egg, to_dir)
408        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
409
410    finally:
411        os.chdir(old_wd)
412    # returning the result
413    log.warn(egg)
414    if not os.path.exists(egg):
415        raise IOError('Could not build the egg.')
416
417
418def _do_download(to_dir, packagename=SETUPTOOLS_PACKAGE):
419    tarball = download_setuptools(packagename, to_dir)
420    version = tarball.split('-')[-1][:-7]
421    egg = os.path.join(to_dir, '%s-%s-py%d.%d.egg'
422                       % (packagename, version, sys.version_info[0], sys.version_info[1]))
423    if not os.path.exists(egg):
424        _build_egg(egg, tarball, to_dir)
425    sys.path.insert(0, egg)
426    import setuptools
427    setuptools.bootstrap_install_from = egg
428
429
430def use_setuptools():
431    # making sure we use the absolute path
432    return _do_download(os.path.abspath(os.curdir))
433
434def download_setuptools(packagename, to_dir):
435    # making sure we use the absolute path
436    to_dir = os.path.abspath(to_dir)
437    try:
438        from urllib.request import urlopen
439    except ImportError:
440        from urllib2 import urlopen
441
442    chksum, url = get_pypi_src_download(packagename)
443    tgz_name = os.path.basename(url)
444    saveto = os.path.join(to_dir, tgz_name)
445
446    src = dst = None
447    if not os.path.exists(saveto):  # Avoid repeated downloads
448        try:
449            log.warn("Downloading %s", url)
450            src = urlopen(url)
451            # Read/write all in one block, so we don't create a corrupt file
452            # if the download is interrupted.
453            data = src.read()
454
455            if chksum is not None:
456                data_sum = md5(data).hexdigest()
457                if data_sum != chksum:
458                    raise RuntimeError("Downloading %s failed: corrupt checksum"%(url,))
459
460
461            dst = open(saveto, "wb")
462            dst.write(data)
463        finally:
464            if src:
465                src.close()
466            if dst:
467                dst.close()
468    return os.path.realpath(saveto)
469
470
471
472def _extractall(self, path=".", members=None):
473    """Extract all members from the archive to the current working
474       directory and set owner, modification time and permissions on
475       directories afterwards. `path' specifies a different directory
476       to extract to. `members' is optional and must be a subset of the
477       list returned by getmembers().
478    """
479    import copy
480    import operator
481    from tarfile import ExtractError
482    directories = []
483
484    if members is None:
485        members = self
486
487    for tarinfo in members:
488        if tarinfo.isdir():
489            # Extract directories with a safe mode.
490            directories.append(tarinfo)
491            tarinfo = copy.copy(tarinfo)
492            tarinfo.mode = 448 # decimal for oct 0700
493        self.extract(tarinfo, path)
494
495    # Reverse sort directories.
496    if sys.version_info < (2, 4):
497        def sorter(dir1, dir2):
498            return cmp(dir1.name, dir2.name)
499        directories.sort(sorter)
500        directories.reverse()
501    else:
502        directories.sort(key=operator.attrgetter('name'), reverse=True)
503
504    # Set correct owner, mtime and filemode on directories.
505    for tarinfo in directories:
506        dirpath = os.path.join(path, tarinfo.name)
507        try:
508            self.chown(tarinfo, dirpath)
509            self.utime(tarinfo, dirpath)
510            self.chmod(tarinfo, dirpath)
511        except ExtractError:
512            e = sys.exc_info()[1]
513            if self.errorlevel > 1:
514                raise
515            else:
516                self._dbg(1, "tarfile: %s" % e)
517
518
519#
520#
521#
522# Definitions of custom commands
523#
524#
525#
526
527try:
528    import setuptools
529
530except ImportError:
531    use_setuptools()
532
533from setuptools import setup
534
535try:
536    from distutils.core import PyPIRCCommand
537except ImportError:
538    PyPIRCCommand = None # Ancient python version
539
540from distutils.core import Command
541from distutils.errors  import DistutilsError
542from distutils import log
543
544if PyPIRCCommand is None:
545    class upload_docs (Command):
546        description = "upload sphinx documentation"
547        user_options = []
548
549        def initialize_options(self):
550            pass
551
552        def finalize_options(self):
553            pass
554
555        def run(self):
556            raise DistutilsError("not supported on this version of python")
557
558else:
559    class upload_docs (PyPIRCCommand):
560        description = "upload sphinx documentation"
561        user_options = PyPIRCCommand.user_options
562
563        def initialize_options(self):
564            PyPIRCCommand.initialize_options(self)
565            self.username = ''
566            self.password = ''
567
568
569        def finalize_options(self):
570            PyPIRCCommand.finalize_options(self)
571            config = self._read_pypirc()
572            if config != {}:
573                self.username = config['username']
574                self.password = config['password']
575
576
577        def run(self):
578            import subprocess
579            import shutil
580            import zipfile
581            import os
582            import urllib
583            import StringIO
584            from base64 import standard_b64encode
585            import httplib
586            import urlparse
587
588            # Extract the package name from distutils metadata
589            meta = self.distribution.metadata
590            name = meta.get_name()
591
592            # Run sphinx
593            if os.path.exists('doc/_build'):
594                shutil.rmtree('doc/_build')
595            os.mkdir('doc/_build')
596
597            p = subprocess.Popen(['make', 'html'],
598                cwd='doc')
599            exit = p.wait()
600            if exit != 0:
601                raise DistutilsError("sphinx-build failed")
602
603            # Collect sphinx output
604            if not os.path.exists('dist'):
605                os.mkdir('dist')
606            zf = zipfile.ZipFile('dist/%s-docs.zip'%(name,), 'w',
607                    compression=zipfile.ZIP_DEFLATED)
608
609            for toplevel, dirs, files in os.walk('doc/_build/html'):
610                for fn in files:
611                    fullname = os.path.join(toplevel, fn)
612                    relname = os.path.relpath(fullname, 'doc/_build/html')
613
614                    print ("%s -> %s"%(fullname, relname))
615
616                    zf.write(fullname, relname)
617
618            zf.close()
619
620            # Upload the results, this code is based on the distutils
621            # 'upload' command.
622            content = open('dist/%s-docs.zip'%(name,), 'rb').read()
623
624            data = {
625                ':action': 'doc_upload',
626                'name': name,
627                'content': ('%s-docs.zip'%(name,), content),
628            }
629            auth = "Basic " + standard_b64encode(self.username + ":" +
630                 self.password)
631
632
633            boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
634            sep_boundary = '\n--' + boundary
635            end_boundary = sep_boundary + '--'
636            body = StringIO.StringIO()
637            for key, value in data.items():
638                if not isinstance(value, list):
639                    value = [value]
640
641                for value in value:
642                    if isinstance(value, tuple):
643                        fn = ';filename="%s"'%(value[0])
644                        value = value[1]
645                    else:
646                        fn = ''
647
648                    body.write(sep_boundary)
649                    body.write('\nContent-Disposition: form-data; name="%s"'%key)
650                    body.write(fn)
651                    body.write("\n\n")
652                    body.write(value)
653
654            body.write(end_boundary)
655            body.write('\n')
656            body = body.getvalue()
657
658            self.announce("Uploading documentation to %s"%(self.repository,), log.INFO)
659
660            schema, netloc, url, params, query, fragments = \
661                    urlparse.urlparse(self.repository)
662
663
664            if schema == 'http':
665                http = httplib.HTTPConnection(netloc)
666            elif schema == 'https':
667                http = httplib.HTTPSConnection(netloc)
668            else:
669                raise AssertionError("unsupported schema "+schema)
670
671            data = ''
672            loglevel = log.INFO
673            try:
674                http.connect()
675                http.putrequest("POST", url)
676                http.putheader('Content-type',
677                    'multipart/form-data; boundary=%s'%boundary)
678                http.putheader('Content-length', str(len(body)))
679                http.putheader('Authorization', auth)
680                http.endheaders()
681                http.send(body)
682            except socket.error:
683                e = socket.exc_info()[1]
684                self.announce(str(e), log.ERROR)
685                return
686
687            r = http.getresponse()
688            if r.status in (200, 301):
689                self.announce('Upload succeeded (%s): %s' % (r.status, r.reason),
690                    log.INFO)
691            else:
692                self.announce('Upload failed (%s): %s' % (r.status, r.reason),
693                    log.ERROR)
694
695                print ('-'*75)
696                print (r.read())
697                print ('-'*75)
698
699
700def recursiveGlob(root, pathPattern):
701    """
702    Recursively look for files matching 'pathPattern'. Return a list
703    of matching files/directories.
704    """
705    result = []
706
707    for rootpath, dirnames, filenames in os.walk(root):
708        for fn in filenames:
709            if fnmatch(fn, pathPattern):
710                result.append(os.path.join(rootpath, fn))
711    return result
712
713
714def importExternalTestCases(unittest,
715        pathPattern="test_*.py", root=".", package=None):
716    """
717    Import all unittests in the PyObjC tree starting at 'root'
718    """
719
720    testFiles = recursiveGlob(root, pathPattern)
721    testModules = map(lambda x:x[len(root)+1:-3].replace('/', '.'), testFiles)
722    if package is not None:
723        testModules = [(package + '.' + m) for m in testModules]
724
725    suites = []
726
727    for modName in testModules:
728        try:
729            module = __import__(modName)
730        except ImportError:
731            print("SKIP %s: %s"%(modName, sys.exc_info()[1]))
732            continue
733
734        if '.' in modName:
735            for elem in modName.split('.')[1:]:
736                module = getattr(module, elem)
737
738        s = unittest.defaultTestLoader.loadTestsFromModule(module)
739        suites.append(s)
740
741    return unittest.TestSuite(suites)
742
743
744
745class test (Command):
746    description = "run test suite"
747    user_options = [
748        ('verbosity=', None, "print what tests are run"),
749    ]
750
751    def initialize_options(self):
752        self.verbosity='1'
753
754    def finalize_options(self):
755        if isinstance(self.verbosity, str):
756            self.verbosity = int(self.verbosity)
757
758
759    def cleanup_environment(self):
760        ei_cmd = self.get_finalized_command('egg_info')
761        egg_name = ei_cmd.egg_name.replace('-', '_')
762
763        to_remove =  []
764        for dirname in sys.path:
765            bn = os.path.basename(dirname)
766            if bn.startswith(egg_name + "-"):
767                to_remove.append(dirname)
768
769        for dirname in to_remove:
770            log.info("removing installed %r from sys.path before testing"%(
771                dirname,))
772            sys.path.remove(dirname)
773
774    def add_project_to_sys_path(self):
775        from pkg_resources import normalize_path, add_activation_listener
776        from pkg_resources import working_set, require
777
778        self.reinitialize_command('egg_info')
779        self.run_command('egg_info')
780        self.reinitialize_command('build_ext', inplace=1)
781        self.run_command('build_ext')
782
783
784        # Check if this distribution is already on sys.path
785        # and remove that version, this ensures that the right
786        # copy of the package gets tested.
787
788        self.__old_path = sys.path[:]
789        self.__old_modules = sys.modules.copy()
790
791
792        ei_cmd = self.get_finalized_command('egg_info')
793        sys.path.insert(0, normalize_path(ei_cmd.egg_base))
794        sys.path.insert(1, os.path.dirname(__file__))
795
796        # Strip the namespace packages defined in this distribution
797        # from sys.modules, needed to reset the search path for
798        # those modules.
799
800        nspkgs = getattr(self.distribution, 'namespace_packages')
801        if nspkgs is not None:
802            for nm in nspkgs:
803                del sys.modules[nm]
804
805        # Reset pkg_resources state:
806        add_activation_listener(lambda dist: dist.activate())
807        working_set.__init__()
808        require('%s==%s'%(ei_cmd.egg_name, ei_cmd.egg_version))
809
810    def remove_from_sys_path(self):
811        from pkg_resources import working_set
812        sys.path[:] = self.__old_path
813        sys.modules.clear()
814        sys.modules.update(self.__old_modules)
815        working_set.__init__()
816
817
818    def run(self):
819        import unittest
820
821        # Ensure that build directory is on sys.path (py3k)
822
823        self.cleanup_environment()
824        self.add_project_to_sys_path()
825
826        try:
827            meta = self.distribution.metadata
828            name = meta.get_name()
829            test_pkg = name + "_tests"
830            suite = importExternalTestCases(unittest,
831                    "test_*.py", test_pkg, test_pkg)
832
833            runner = unittest.TextTestRunner(verbosity=self.verbosity)
834            result = runner.run(suite)
835
836            # Print out summary. This is a structured format that
837            # should make it easy to use this information in scripts.
838            summary = dict(
839                count=result.testsRun,
840                fails=len(result.failures),
841                errors=len(result.errors),
842                xfails=len(getattr(result, 'expectedFailures', [])),
843                xpass=len(getattr(result, 'expectedSuccesses', [])),
844                skip=len(getattr(result, 'skipped', [])),
845            )
846            print("SUMMARY: %s"%(summary,))
847
848        finally:
849            self.remove_from_sys_path()
850
851#
852#
853#
854#  And finally run the setuptools main entry point.
855#
856#
857#
858
859metadata = parse_setup_cfg()
860
861setup(
862    cmdclass=dict(
863        upload_docs=upload_docs,
864        test=test,
865    ),
866    **metadata
867)
868