1"""Tests for distutils.command.sdist."""
2import os
3import tarfile
4import unittest
5import warnings
6import zipfile
7from os.path import join
8from textwrap import dedent
9from test.test_support import captured_stdout, check_warnings, run_unittest
10
11# zlib is not used here, but if it's not available
12# the tests that use zipfile may fail
13try:
14    import zlib
15except ImportError:
16    zlib = None
17
18try:
19    import grp
20    import pwd
21    UID_GID_SUPPORT = True
22except ImportError:
23    UID_GID_SUPPORT = False
24
25
26from distutils.command.sdist import sdist, show_formats
27from distutils.core import Distribution
28from distutils.tests.test_config import PyPIRCCommandTestCase
29from distutils.errors import DistutilsOptionError
30from distutils.spawn import find_executable
31from distutils.log import WARN
32from distutils.filelist import FileList
33from distutils.archive_util import ARCHIVE_FORMATS
34
35SETUP_PY = """
36from distutils.core import setup
37import somecode
38
39setup(name='fake')
40"""
41
42MANIFEST = """\
43# file GENERATED by distutils, do NOT edit
44README
45buildout.cfg
46inroot.txt
47setup.py
48data%(sep)sdata.dt
49scripts%(sep)sscript.py
50some%(sep)sfile.txt
51some%(sep)sother_file.txt
52somecode%(sep)s__init__.py
53somecode%(sep)sdoc.dat
54somecode%(sep)sdoc.txt
55"""
56
57class SDistTestCase(PyPIRCCommandTestCase):
58
59    def setUp(self):
60        # PyPIRCCommandTestCase creates a temp dir already
61        # and put it in self.tmp_dir
62        super(SDistTestCase, self).setUp()
63        # setting up an environment
64        self.old_path = os.getcwd()
65        os.mkdir(join(self.tmp_dir, 'somecode'))
66        os.mkdir(join(self.tmp_dir, 'dist'))
67        # a package, and a README
68        self.write_file((self.tmp_dir, 'README'), 'xxx')
69        self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#')
70        self.write_file((self.tmp_dir, 'setup.py'), SETUP_PY)
71        os.chdir(self.tmp_dir)
72
73    def tearDown(self):
74        # back to normal
75        os.chdir(self.old_path)
76        super(SDistTestCase, self).tearDown()
77
78    def get_cmd(self, metadata=None):
79        """Returns a cmd"""
80        if metadata is None:
81            metadata = {'name': 'fake', 'version': '1.0',
82                        'url': 'xxx', 'author': 'xxx',
83                        'author_email': 'xxx'}
84        dist = Distribution(metadata)
85        dist.script_name = 'setup.py'
86        dist.packages = ['somecode']
87        dist.include_package_data = True
88        cmd = sdist(dist)
89        cmd.dist_dir = 'dist'
90        return dist, cmd
91
92    @unittest.skipUnless(zlib, "requires zlib")
93    def test_prune_file_list(self):
94        # this test creates a project with some VCS dirs and an NFS rename
95        # file, then launches sdist to check they get pruned on all systems
96
97        # creating VCS directories with some files in them
98        os.mkdir(join(self.tmp_dir, 'somecode', '.svn'))
99        self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx')
100
101        os.mkdir(join(self.tmp_dir, 'somecode', '.hg'))
102        self.write_file((self.tmp_dir, 'somecode', '.hg',
103                         'ok'), 'xxx')
104
105        os.mkdir(join(self.tmp_dir, 'somecode', '.git'))
106        self.write_file((self.tmp_dir, 'somecode', '.git',
107                         'ok'), 'xxx')
108
109        self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx')
110
111        # now building a sdist
112        dist, cmd = self.get_cmd()
113
114        # zip is available universally
115        # (tar might not be installed under win32)
116        cmd.formats = ['zip']
117
118        cmd.ensure_finalized()
119        cmd.run()
120
121        # now let's check what we have
122        dist_folder = join(self.tmp_dir, 'dist')
123        files = os.listdir(dist_folder)
124        self.assertEqual(files, ['fake-1.0.zip'])
125
126        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
127        try:
128            content = zip_file.namelist()
129        finally:
130            zip_file.close()
131
132        # making sure everything has been pruned correctly
133        self.assertEqual(len(content), 4)
134
135    @unittest.skipUnless(zlib, "requires zlib")
136    def test_make_distribution(self):
137
138        # check if tar and gzip are installed
139        if (find_executable('tar') is None or
140            find_executable('gzip') is None):
141            return
142
143        # now building a sdist
144        dist, cmd = self.get_cmd()
145
146        # creating a gztar then a tar
147        cmd.formats = ['gztar', 'tar']
148        cmd.ensure_finalized()
149        cmd.run()
150
151        # making sure we have two files
152        dist_folder = join(self.tmp_dir, 'dist')
153        result = os.listdir(dist_folder)
154        result.sort()
155        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
156
157        os.remove(join(dist_folder, 'fake-1.0.tar'))
158        os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
159
160        # now trying a tar then a gztar
161        cmd.formats = ['tar', 'gztar']
162
163        cmd.ensure_finalized()
164        cmd.run()
165
166        result = os.listdir(dist_folder)
167        result.sort()
168        self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
169
170    @unittest.skipUnless(zlib, "requires zlib")
171    def test_unicode_metadata_tgz(self):
172        """
173        Unicode name or version should not break building to tar.gz format.
174        Reference issue #11638.
175        """
176
177        # create the sdist command with unicode parameters
178        dist, cmd = self.get_cmd({'name': u'fake', 'version': u'1.0'})
179
180        # create the sdist as gztar and run the command
181        cmd.formats = ['gztar']
182        cmd.ensure_finalized()
183        cmd.run()
184
185        # The command should have created the .tar.gz file
186        dist_folder = join(self.tmp_dir, 'dist')
187        result = os.listdir(dist_folder)
188        self.assertEqual(result, ['fake-1.0.tar.gz'])
189
190        os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
191
192    @unittest.skipUnless(zlib, "requires zlib")
193    def test_add_defaults(self):
194
195        # http://bugs.python.org/issue2279
196
197        # add_default should also include
198        # data_files and package_data
199        dist, cmd = self.get_cmd()
200
201        # filling data_files by pointing files
202        # in package_data
203        dist.package_data = {'': ['*.cfg', '*.dat'],
204                             'somecode': ['*.txt']}
205        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
206        self.write_file((self.tmp_dir, 'somecode', 'doc.dat'), '#')
207
208        # adding some data in data_files
209        data_dir = join(self.tmp_dir, 'data')
210        os.mkdir(data_dir)
211        self.write_file((data_dir, 'data.dt'), '#')
212        some_dir = join(self.tmp_dir, 'some')
213        os.mkdir(some_dir)
214        # make sure VCS directories are pruned (#14004)
215        hg_dir = join(self.tmp_dir, '.hg')
216        os.mkdir(hg_dir)
217        self.write_file((hg_dir, 'last-message.txt'), '#')
218        # a buggy regex used to prevent this from working on windows (#6884)
219        self.write_file((self.tmp_dir, 'buildout.cfg'), '#')
220        self.write_file((self.tmp_dir, 'inroot.txt'), '#')
221        self.write_file((some_dir, 'file.txt'), '#')
222        self.write_file((some_dir, 'other_file.txt'), '#')
223
224        dist.data_files = [('data', ['data/data.dt',
225                                     'buildout.cfg',
226                                     'inroot.txt',
227                                     'notexisting']),
228                           'some/file.txt',
229                           'some/other_file.txt']
230
231        # adding a script
232        script_dir = join(self.tmp_dir, 'scripts')
233        os.mkdir(script_dir)
234        self.write_file((script_dir, 'script.py'), '#')
235        dist.scripts = [join('scripts', 'script.py')]
236
237        cmd.formats = ['zip']
238        cmd.use_defaults = True
239
240        cmd.ensure_finalized()
241        cmd.run()
242
243        # now let's check what we have
244        dist_folder = join(self.tmp_dir, 'dist')
245        files = os.listdir(dist_folder)
246        self.assertEqual(files, ['fake-1.0.zip'])
247
248        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
249        try:
250            content = zip_file.namelist()
251        finally:
252            zip_file.close()
253
254        # making sure everything was added
255        self.assertEqual(len(content), 12)
256
257        # checking the MANIFEST
258        f = open(join(self.tmp_dir, 'MANIFEST'))
259        try:
260            manifest = f.read()
261        finally:
262            f.close()
263        self.assertEqual(manifest, MANIFEST % {'sep': os.sep})
264
265    @unittest.skipUnless(zlib, "requires zlib")
266    def test_metadata_check_option(self):
267        # testing the `medata-check` option
268        dist, cmd = self.get_cmd(metadata={})
269
270        # this should raise some warnings !
271        # with the `check` subcommand
272        cmd.ensure_finalized()
273        cmd.run()
274        warnings = [msg for msg in self.get_logs(WARN) if
275                    msg.startswith('warning: check:')]
276        self.assertEqual(len(warnings), 2)
277
278        # trying with a complete set of metadata
279        self.clear_logs()
280        dist, cmd = self.get_cmd()
281        cmd.ensure_finalized()
282        cmd.metadata_check = 0
283        cmd.run()
284        warnings = [msg for msg in self.get_logs(WARN) if
285                    msg.startswith('warning: check:')]
286        self.assertEqual(len(warnings), 0)
287
288    def test_check_metadata_deprecated(self):
289        # makes sure make_metadata is deprecated
290        dist, cmd = self.get_cmd()
291        with check_warnings() as w:
292            warnings.simplefilter("always")
293            cmd.check_metadata()
294            self.assertEqual(len(w.warnings), 1)
295
296    def test_show_formats(self):
297        with captured_stdout() as stdout:
298            show_formats()
299
300        # the output should be a header line + one line per format
301        num_formats = len(ARCHIVE_FORMATS.keys())
302        output = [line for line in stdout.getvalue().split('\n')
303                  if line.strip().startswith('--formats=')]
304        self.assertEqual(len(output), num_formats)
305
306    def test_finalize_options(self):
307        dist, cmd = self.get_cmd()
308        cmd.finalize_options()
309
310        # default options set by finalize
311        self.assertEqual(cmd.manifest, 'MANIFEST')
312        self.assertEqual(cmd.template, 'MANIFEST.in')
313        self.assertEqual(cmd.dist_dir, 'dist')
314
315        # formats has to be a string splitable on (' ', ',') or
316        # a stringlist
317        cmd.formats = 1
318        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
319        cmd.formats = ['zip']
320        cmd.finalize_options()
321
322        # formats has to be known
323        cmd.formats = 'supazipa'
324        self.assertRaises(DistutilsOptionError, cmd.finalize_options)
325
326    @unittest.skipUnless(zlib, "requires zlib")
327    @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
328    def test_make_distribution_owner_group(self):
329
330        # check if tar and gzip are installed
331        if (find_executable('tar') is None or
332            find_executable('gzip') is None):
333            return
334
335        # now building a sdist
336        dist, cmd = self.get_cmd()
337
338        # creating a gztar and specifying the owner+group
339        cmd.formats = ['gztar']
340        cmd.owner = pwd.getpwuid(0)[0]
341        cmd.group = grp.getgrgid(0)[0]
342        cmd.ensure_finalized()
343        cmd.run()
344
345        # making sure we have the good rights
346        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
347        archive = tarfile.open(archive_name)
348        try:
349            for member in archive.getmembers():
350                self.assertEqual(member.uid, 0)
351                self.assertEqual(member.gid, 0)
352        finally:
353            archive.close()
354
355        # building a sdist again
356        dist, cmd = self.get_cmd()
357
358        # creating a gztar
359        cmd.formats = ['gztar']
360        cmd.ensure_finalized()
361        cmd.run()
362
363        # making sure we have the good rights
364        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
365        archive = tarfile.open(archive_name)
366
367        # note that we are not testing the group ownership here
368        # because, depending on the platforms and the container
369        # rights (see #7408)
370        try:
371            for member in archive.getmembers():
372                self.assertEqual(member.uid, os.getuid())
373        finally:
374            archive.close()
375
376    # the following tests make sure there is a nice error message instead
377    # of a traceback when parsing an invalid manifest template
378
379    def _check_template(self, content):
380        dist, cmd = self.get_cmd()
381        os.chdir(self.tmp_dir)
382        self.write_file('MANIFEST.in', content)
383        cmd.ensure_finalized()
384        cmd.filelist = FileList()
385        cmd.read_template()
386        warnings = self.get_logs(WARN)
387        self.assertEqual(len(warnings), 1)
388
389    def test_invalid_template_unknown_command(self):
390        self._check_template('taunt knights *')
391
392    def test_invalid_template_wrong_arguments(self):
393        # this manifest command takes one argument
394        self._check_template('prune')
395
396    @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only')
397    def test_invalid_template_wrong_path(self):
398        # on Windows, trailing slashes are not allowed
399        # this used to crash instead of raising a warning: #8286
400        self._check_template('include examples/')
401
402    @unittest.skipUnless(zlib, "requires zlib")
403    def test_get_file_list(self):
404        # make sure MANIFEST is recalculated
405        dist, cmd = self.get_cmd()
406
407        # filling data_files by pointing files in package_data
408        dist.package_data = {'somecode': ['*.txt']}
409        self.write_file((self.tmp_dir, 'somecode', 'doc.txt'), '#')
410        cmd.formats = ['gztar']
411        cmd.ensure_finalized()
412        cmd.run()
413
414        f = open(cmd.manifest)
415        try:
416            manifest = [line.strip() for line in f.read().split('\n')
417                        if line.strip() != '']
418        finally:
419            f.close()
420
421        self.assertEqual(len(manifest), 5)
422
423        # adding a file
424        self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#')
425
426        # make sure build_py is reinitialized, like a fresh run
427        build_py = dist.get_command_obj('build_py')
428        build_py.finalized = False
429        build_py.ensure_finalized()
430
431        cmd.run()
432
433        f = open(cmd.manifest)
434        try:
435            manifest2 = [line.strip() for line in f.read().split('\n')
436                         if line.strip() != '']
437        finally:
438            f.close()
439
440        # do we have the new file in MANIFEST ?
441        self.assertEqual(len(manifest2), 6)
442        self.assertIn('doc2.txt', manifest2[-1])
443
444    @unittest.skipUnless(zlib, "requires zlib")
445    def test_manifest_marker(self):
446        # check that autogenerated MANIFESTs have a marker
447        dist, cmd = self.get_cmd()
448        cmd.ensure_finalized()
449        cmd.run()
450
451        f = open(cmd.manifest)
452        try:
453            manifest = [line.strip() for line in f.read().split('\n')
454                        if line.strip() != '']
455        finally:
456            f.close()
457
458        self.assertEqual(manifest[0],
459                         '# file GENERATED by distutils, do NOT edit')
460
461    @unittest.skipUnless(zlib, 'requires zlib')
462    def test_manifest_comments(self):
463        # make sure comments don't cause exceptions or wrong includes
464        contents = dedent("""\
465            # bad.py
466            #bad.py
467            good.py
468            """)
469        dist, cmd = self.get_cmd()
470        cmd.ensure_finalized()
471        self.write_file((self.tmp_dir, cmd.manifest), contents)
472        self.write_file((self.tmp_dir, 'good.py'), '# pick me!')
473        self.write_file((self.tmp_dir, 'bad.py'), "# don't pick me!")
474        self.write_file((self.tmp_dir, '#bad.py'), "# don't pick me!")
475        cmd.run()
476        self.assertEqual(cmd.filelist.files, ['good.py'])
477
478    @unittest.skipUnless(zlib, "requires zlib")
479    def test_manual_manifest(self):
480        # check that a MANIFEST without a marker is left alone
481        dist, cmd = self.get_cmd()
482        cmd.formats = ['gztar']
483        cmd.ensure_finalized()
484        self.write_file((self.tmp_dir, cmd.manifest), 'README.manual')
485        self.write_file((self.tmp_dir, 'README.manual'),
486                         'This project maintains its MANIFEST file itself.')
487        cmd.run()
488        self.assertEqual(cmd.filelist.files, ['README.manual'])
489
490        f = open(cmd.manifest)
491        try:
492            manifest = [line.strip() for line in f.read().split('\n')
493                        if line.strip() != '']
494        finally:
495            f.close()
496
497        self.assertEqual(manifest, ['README.manual'])
498
499        archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
500        archive = tarfile.open(archive_name)
501        try:
502            filenames = [tarinfo.name for tarinfo in archive]
503        finally:
504            archive.close()
505        self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO',
506                                             'fake-1.0/README.manual'])
507
508def test_suite():
509    return unittest.makeSuite(SDistTestCase)
510
511if __name__ == "__main__":
512    run_unittest(test_suite())
513