1# Tests invocation of the interpreter with various command line arguments
2# Most tests are executed with environment variables ignored
3# See test_cmd_line_script.py for testing of script execution
4
5import test.support, unittest
6import os
7import shutil
8import sys
9import subprocess
10import tempfile
11from test.support import script_helper, is_android
12from test.support.script_helper import (spawn_python, kill_python, assert_python_ok,
13    assert_python_failure)
14
15
16# XXX (ncoghlan): Move to script_helper and make consistent with run_python
17def _kill_python_and_exit_code(p):
18    data = kill_python(p)
19    returncode = p.wait()
20    return data, returncode
21
22class CmdLineTest(unittest.TestCase):
23    def test_directories(self):
24        assert_python_failure('.')
25        assert_python_failure('< .')
26
27    def verify_valid_flag(self, cmd_line):
28        rc, out, err = assert_python_ok(*cmd_line)
29        self.assertTrue(out == b'' or out.endswith(b'\n'))
30        self.assertNotIn(b'Traceback', out)
31        self.assertNotIn(b'Traceback', err)
32
33    def test_optimize(self):
34        self.verify_valid_flag('-O')
35        self.verify_valid_flag('-OO')
36
37    def test_site_flag(self):
38        self.verify_valid_flag('-S')
39
40    def test_usage(self):
41        rc, out, err = assert_python_ok('-h')
42        self.assertIn(b'usage', out)
43
44    def test_version(self):
45        version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii")
46        for switch in '-V', '--version', '-VV':
47            rc, out, err = assert_python_ok(switch)
48            self.assertFalse(err.startswith(version))
49            self.assertTrue(out.startswith(version))
50
51    def test_verbose(self):
52        # -v causes imports to write to stderr.  If the write to
53        # stderr itself causes an import to happen (for the output
54        # codec), a recursion loop can occur.
55        rc, out, err = assert_python_ok('-v')
56        self.assertNotIn(b'stack overflow', err)
57        rc, out, err = assert_python_ok('-vv')
58        self.assertNotIn(b'stack overflow', err)
59
60    def test_xoptions(self):
61        def get_xoptions(*args):
62            # use subprocess module directly because test.support.script_helper adds
63            # "-X faulthandler" to the command line
64            args = (sys.executable, '-E') + args
65            args += ('-c', 'import sys; print(sys._xoptions)')
66            out = subprocess.check_output(args)
67            opts = eval(out.splitlines()[0])
68            return opts
69
70        opts = get_xoptions()
71        self.assertEqual(opts, {})
72
73        opts = get_xoptions('-Xa', '-Xb=c,d=e')
74        self.assertEqual(opts, {'a': True, 'b': 'c,d=e'})
75
76    def test_showrefcount(self):
77        def run_python(*args):
78            # this is similar to assert_python_ok but doesn't strip
79            # the refcount from stderr.  It can be replaced once
80            # assert_python_ok stops doing that.
81            cmd = [sys.executable]
82            cmd.extend(args)
83            PIPE = subprocess.PIPE
84            p = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE)
85            out, err = p.communicate()
86            p.stdout.close()
87            p.stderr.close()
88            rc = p.returncode
89            self.assertEqual(rc, 0)
90            return rc, out, err
91        code = 'import sys; print(sys._xoptions)'
92        # normally the refcount is hidden
93        rc, out, err = run_python('-c', code)
94        self.assertEqual(out.rstrip(), b'{}')
95        self.assertEqual(err, b'')
96        # "-X showrefcount" shows the refcount, but only in debug builds
97        rc, out, err = run_python('-X', 'showrefcount', '-c', code)
98        self.assertEqual(out.rstrip(), b"{'showrefcount': True}")
99        if hasattr(sys, 'gettotalrefcount'):  # debug build
100            self.assertRegex(err, br'^\[\d+ refs, \d+ blocks\]')
101        else:
102            self.assertEqual(err, b'')
103
104    def test_run_module(self):
105        # Test expected operation of the '-m' switch
106        # Switch needs an argument
107        assert_python_failure('-m')
108        # Check we get an error for a nonexistent module
109        assert_python_failure('-m', 'fnord43520xyz')
110        # Check the runpy module also gives an error for
111        # a nonexistent module
112        assert_python_failure('-m', 'runpy', 'fnord43520xyz')
113        # All good if module is located and run successfully
114        assert_python_ok('-m', 'timeit', '-n', '1')
115
116    def test_run_module_bug1764407(self):
117        # -m and -i need to play well together
118        # Runs the timeit module and checks the __main__
119        # namespace has been populated appropriately
120        p = spawn_python('-i', '-m', 'timeit', '-n', '1')
121        p.stdin.write(b'Timer\n')
122        p.stdin.write(b'exit()\n')
123        data = kill_python(p)
124        self.assertTrue(data.find(b'1 loop') != -1)
125        self.assertTrue(data.find(b'__main__.Timer') != -1)
126
127    def test_run_code(self):
128        # Test expected operation of the '-c' switch
129        # Switch needs an argument
130        assert_python_failure('-c')
131        # Check we get an error for an uncaught exception
132        assert_python_failure('-c', 'raise Exception')
133        # All good if execution is successful
134        assert_python_ok('-c', 'pass')
135
136    @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII')
137    def test_non_ascii(self):
138        # Test handling of non-ascii data
139        command = ("assert(ord(%r) == %s)"
140                   % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII)))
141        assert_python_ok('-c', command)
142
143    # On Windows, pass bytes to subprocess doesn't test how Python decodes the
144    # command line, but how subprocess does decode bytes to unicode. Python
145    # doesn't decode the command line because Windows provides directly the
146    # arguments as unicode (using wmain() instead of main()).
147    @unittest.skipIf(sys.platform == 'win32',
148                     'Windows has a native unicode API')
149    def test_undecodable_code(self):
150        undecodable = b"\xff"
151        env = os.environ.copy()
152        # Use C locale to get ascii for the locale encoding
153        env['LC_ALL'] = 'C'
154        code = (
155            b'import locale; '
156            b'print(ascii("' + undecodable + b'"), '
157                b'locale.getpreferredencoding())')
158        p = subprocess.Popen(
159            [sys.executable, "-c", code],
160            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
161            env=env)
162        stdout, stderr = p.communicate()
163        if p.returncode == 1:
164            # _Py_char2wchar() decoded b'\xff' as '\udcff' (b'\xff' is not
165            # decodable from ASCII) and run_command() failed on
166            # PyUnicode_AsUTF8String(). This is the expected behaviour on
167            # Linux.
168            pattern = b"Unable to decode the command from the command line:"
169        elif p.returncode == 0:
170            # _Py_char2wchar() decoded b'\xff' as '\xff' even if the locale is
171            # C and the locale encoding is ASCII. It occurs on FreeBSD, Solaris
172            # and Mac OS X.
173            pattern = b"'\\xff' "
174            # The output is followed by the encoding name, an alias to ASCII.
175            # Examples: "US-ASCII" or "646" (ISO 646, on Solaris).
176        else:
177            raise AssertionError("Unknown exit code: %s, output=%a" % (p.returncode, stdout))
178        if not stdout.startswith(pattern):
179            raise AssertionError("%a doesn't start with %a" % (stdout, pattern))
180
181    @unittest.skipUnless((sys.platform == 'darwin' or
182                is_android), 'test specific to Mac OS X and Android')
183    def test_osx_android_utf8(self):
184        def check_output(text):
185            decoded = text.decode('utf-8', 'surrogateescape')
186            expected = ascii(decoded).encode('ascii') + b'\n'
187
188            env = os.environ.copy()
189            # C locale gives ASCII locale encoding, but Python uses UTF-8
190            # to parse the command line arguments on Mac OS X and Android.
191            env['LC_ALL'] = 'C'
192
193            p = subprocess.Popen(
194                (sys.executable, "-c", "import sys; print(ascii(sys.argv[1]))", text),
195                stdout=subprocess.PIPE,
196                env=env)
197            stdout, stderr = p.communicate()
198            self.assertEqual(stdout, expected)
199            self.assertEqual(p.returncode, 0)
200
201        # test valid utf-8
202        text = 'e:\xe9, euro:\u20ac, non-bmp:\U0010ffff'.encode('utf-8')
203        check_output(text)
204
205        # test invalid utf-8
206        text = (
207            b'\xff'         # invalid byte
208            b'\xc3\xa9'     # valid utf-8 character
209            b'\xc3\xff'     # invalid byte sequence
210            b'\xed\xa0\x80' # lone surrogate character (invalid)
211        )
212        check_output(text)
213
214    def test_unbuffered_output(self):
215        # Test expected operation of the '-u' switch
216        for stream in ('stdout', 'stderr'):
217            # Binary is unbuffered
218            code = ("import os, sys; sys.%s.buffer.write(b'x'); os._exit(0)"
219                % stream)
220            rc, out, err = assert_python_ok('-u', '-c', code)
221            data = err if stream == 'stderr' else out
222            self.assertEqual(data, b'x', "binary %s not unbuffered" % stream)
223            # Text is line-buffered
224            code = ("import os, sys; sys.%s.write('x\\n'); os._exit(0)"
225                % stream)
226            rc, out, err = assert_python_ok('-u', '-c', code)
227            data = err if stream == 'stderr' else out
228            self.assertEqual(data.strip(), b'x',
229                "text %s not line-buffered" % stream)
230
231    def test_unbuffered_input(self):
232        # sys.stdin still works with '-u'
233        code = ("import sys; sys.stdout.write(sys.stdin.read(1))")
234        p = spawn_python('-u', '-c', code)
235        p.stdin.write(b'x')
236        p.stdin.flush()
237        data, rc = _kill_python_and_exit_code(p)
238        self.assertEqual(rc, 0)
239        self.assertTrue(data.startswith(b'x'), data)
240
241    def test_large_PYTHONPATH(self):
242        path1 = "ABCDE" * 100
243        path2 = "FGHIJ" * 100
244        path = path1 + os.pathsep + path2
245
246        code = """if 1:
247            import sys
248            path = ":".join(sys.path)
249            path = path.encode("ascii", "backslashreplace")
250            sys.stdout.buffer.write(path)"""
251        rc, out, err = assert_python_ok('-S', '-c', code,
252                                        PYTHONPATH=path)
253        self.assertIn(path1.encode('ascii'), out)
254        self.assertIn(path2.encode('ascii'), out)
255
256    def test_empty_PYTHONPATH_issue16309(self):
257        # On Posix, it is documented that setting PATH to the
258        # empty string is equivalent to not setting PATH at all,
259        # which is an exception to the rule that in a string like
260        # "/bin::/usr/bin" the empty string in the middle gets
261        # interpreted as '.'
262        code = """if 1:
263            import sys
264            path = ":".join(sys.path)
265            path = path.encode("ascii", "backslashreplace")
266            sys.stdout.buffer.write(path)"""
267        rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="")
268        rc2, out2, err2 = assert_python_ok('-c', code, __isolated=False)
269        # regarding to Posix specification, outputs should be equal
270        # for empty and unset PYTHONPATH
271        self.assertEqual(out1, out2)
272
273    def test_displayhook_unencodable(self):
274        for encoding in ('ascii', 'latin-1', 'utf-8'):
275            # We are testing a PYTHON environment variable here, so we can't
276            # use -E, -I, or script_helper (which uses them).  So instead we do
277            # poor-man's isolation by deleting the PYTHON vars from env.
278            env = {key:value for (key,value) in os.environ.copy().items()
279                   if not key.startswith('PYTHON')}
280            env['PYTHONIOENCODING'] = encoding
281            p = subprocess.Popen(
282                [sys.executable, '-i'],
283                stdin=subprocess.PIPE,
284                stdout=subprocess.PIPE,
285                stderr=subprocess.STDOUT,
286                env=env)
287            # non-ascii, surrogate, non-BMP printable, non-BMP unprintable
288            text = "a=\xe9 b=\uDC80 c=\U00010000 d=\U0010FFFF"
289            p.stdin.write(ascii(text).encode('ascii') + b"\n")
290            p.stdin.write(b'exit()\n')
291            data = kill_python(p)
292            escaped = repr(text).encode(encoding, 'backslashreplace')
293            self.assertIn(escaped, data)
294
295    def check_input(self, code, expected):
296        with tempfile.NamedTemporaryFile("wb+") as stdin:
297            sep = os.linesep.encode('ASCII')
298            stdin.write(sep.join((b'abc', b'def')))
299            stdin.flush()
300            stdin.seek(0)
301            with subprocess.Popen(
302                (sys.executable, "-c", code),
303                stdin=stdin, stdout=subprocess.PIPE) as proc:
304                stdout, stderr = proc.communicate()
305        self.assertEqual(stdout.rstrip(), expected)
306
307    def test_stdin_readline(self):
308        # Issue #11272: check that sys.stdin.readline() replaces '\r\n' by '\n'
309        # on Windows (sys.stdin is opened in binary mode)
310        self.check_input(
311            "import sys; print(repr(sys.stdin.readline()))",
312            b"'abc\\n'")
313
314    def test_builtin_input(self):
315        # Issue #11272: check that input() strips newlines ('\n' or '\r\n')
316        self.check_input(
317            "print(repr(input()))",
318            b"'abc'")
319
320    def test_output_newline(self):
321        # Issue 13119 Newline for print() should be \r\n on Windows.
322        code = """if 1:
323            import sys
324            print(1)
325            print(2)
326            print(3, file=sys.stderr)
327            print(4, file=sys.stderr)"""
328        rc, out, err = assert_python_ok('-c', code)
329
330        if sys.platform == 'win32':
331            self.assertEqual(b'1\r\n2\r\n', out)
332            self.assertEqual(b'3\r\n4', err)
333        else:
334            self.assertEqual(b'1\n2\n', out)
335            self.assertEqual(b'3\n4', err)
336
337    def test_unmached_quote(self):
338        # Issue #10206: python program starting with unmatched quote
339        # spewed spaces to stdout
340        rc, out, err = assert_python_failure('-c', "'")
341        self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError')
342        self.assertEqual(b'', out)
343
344    def test_stdout_flush_at_shutdown(self):
345        # Issue #5319: if stdout.flush() fails at shutdown, an error should
346        # be printed out.
347        code = """if 1:
348            import os, sys, test.support
349            test.support.SuppressCrashReport().__enter__()
350            sys.stdout.write('x')
351            os.close(sys.stdout.fileno())"""
352        rc, out, err = assert_python_failure('-c', code)
353        self.assertEqual(b'', out)
354        self.assertEqual(120, rc)
355        self.assertRegex(err.decode('ascii', 'ignore'),
356                         'Exception ignored in.*\nOSError: .*')
357
358    def test_closed_stdout(self):
359        # Issue #13444: if stdout has been explicitly closed, we should
360        # not attempt to flush it at shutdown.
361        code = "import sys; sys.stdout.close()"
362        rc, out, err = assert_python_ok('-c', code)
363        self.assertEqual(b'', err)
364
365    # Issue #7111: Python should work without standard streams
366
367    @unittest.skipIf(os.name != 'posix', "test needs POSIX semantics")
368    def _test_no_stdio(self, streams):
369        code = """if 1:
370            import os, sys
371            for i, s in enumerate({streams}):
372                if getattr(sys, s) is not None:
373                    os._exit(i + 1)
374            os._exit(42)""".format(streams=streams)
375        def preexec():
376            if 'stdin' in streams:
377                os.close(0)
378            if 'stdout' in streams:
379                os.close(1)
380            if 'stderr' in streams:
381                os.close(2)
382        p = subprocess.Popen(
383            [sys.executable, "-E", "-c", code],
384            stdin=subprocess.PIPE,
385            stdout=subprocess.PIPE,
386            stderr=subprocess.PIPE,
387            preexec_fn=preexec)
388        out, err = p.communicate()
389        self.assertEqual(test.support.strip_python_stderr(err), b'')
390        self.assertEqual(p.returncode, 42)
391
392    def test_no_stdin(self):
393        self._test_no_stdio(['stdin'])
394
395    def test_no_stdout(self):
396        self._test_no_stdio(['stdout'])
397
398    def test_no_stderr(self):
399        self._test_no_stdio(['stderr'])
400
401    def test_no_std_streams(self):
402        self._test_no_stdio(['stdin', 'stdout', 'stderr'])
403
404    def test_hash_randomization(self):
405        # Verify that -R enables hash randomization:
406        self.verify_valid_flag('-R')
407        hashes = []
408        if os.environ.get('PYTHONHASHSEED', 'random') != 'random':
409            env = dict(os.environ)  # copy
410            # We need to test that it is enabled by default without
411            # the environment variable enabling it for us.
412            del env['PYTHONHASHSEED']
413            env['__cleanenv'] = '1'  # consumed by assert_python_ok()
414        else:
415            env = {}
416        for i in range(3):
417            code = 'print(hash("spam"))'
418            rc, out, err = assert_python_ok('-c', code, **env)
419            self.assertEqual(rc, 0)
420            hashes.append(out)
421        hashes = sorted(set(hashes))  # uniq
422        # Rare chance of failure due to 3 random seeds honestly being equal.
423        self.assertGreater(len(hashes), 1,
424                           msg='3 runs produced an identical random hash '
425                               ' for "spam": {}'.format(hashes))
426
427        # Verify that sys.flags contains hash_randomization
428        code = 'import sys; print("random is", sys.flags.hash_randomization)'
429        rc, out, err = assert_python_ok('-c', code)
430        self.assertEqual(rc, 0)
431        self.assertIn(b'random is 1', out)
432
433    def test_del___main__(self):
434        # Issue #15001: PyRun_SimpleFileExFlags() did crash because it kept a
435        # borrowed reference to the dict of __main__ module and later modify
436        # the dict whereas the module was destroyed
437        filename = test.support.TESTFN
438        self.addCleanup(test.support.unlink, filename)
439        with open(filename, "w") as script:
440            print("import sys", file=script)
441            print("del sys.modules['__main__']", file=script)
442        assert_python_ok(filename)
443
444    def test_unknown_options(self):
445        rc, out, err = assert_python_failure('-E', '-z')
446        self.assertIn(b'Unknown option: -z', err)
447        self.assertEqual(err.splitlines().count(b'Unknown option: -z'), 1)
448        self.assertEqual(b'', out)
449        # Add "without='-E'" to prevent _assert_python to append -E
450        # to env_vars and change the output of stderr
451        rc, out, err = assert_python_failure('-z', without='-E')
452        self.assertIn(b'Unknown option: -z', err)
453        self.assertEqual(err.splitlines().count(b'Unknown option: -z'), 1)
454        self.assertEqual(b'', out)
455        rc, out, err = assert_python_failure('-a', '-z', without='-E')
456        self.assertIn(b'Unknown option: -a', err)
457        # only the first unknown option is reported
458        self.assertNotIn(b'Unknown option: -z', err)
459        self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1)
460        self.assertEqual(b'', out)
461
462    @unittest.skipIf(script_helper.interpreter_requires_environment(),
463                     'Cannot run -I tests when PYTHON env vars are required.')
464    def test_isolatedmode(self):
465        self.verify_valid_flag('-I')
466        self.verify_valid_flag('-IEs')
467        rc, out, err = assert_python_ok('-I', '-c',
468            'from sys import flags as f; '
469            'print(f.no_user_site, f.ignore_environment, f.isolated)',
470            # dummyvar to prevent extraneous -E
471            dummyvar="")
472        self.assertEqual(out.strip(), b'1 1 1')
473        with test.support.temp_cwd() as tmpdir:
474            fake = os.path.join(tmpdir, "uuid.py")
475            main = os.path.join(tmpdir, "main.py")
476            with open(fake, "w") as f:
477                f.write("raise RuntimeError('isolated mode test')\n")
478            with open(main, "w") as f:
479                f.write("import uuid\n")
480                f.write("print('ok')\n")
481            self.assertRaises(subprocess.CalledProcessError,
482                              subprocess.check_output,
483                              [sys.executable, main], cwd=tmpdir,
484                              stderr=subprocess.DEVNULL)
485            out = subprocess.check_output([sys.executable, "-I", main],
486                                          cwd=tmpdir)
487            self.assertEqual(out.strip(), b"ok")
488
489def test_main():
490    test.support.run_unittest(CmdLineTest)
491    test.support.reap_children()
492
493if __name__ == "__main__":
494    test_main()
495