1#./python
2"""Run Python tests with multiple installations of OpenSSL
3
4The script
5
6  (1) downloads OpenSSL tar bundle
7  (2) extracts it to ../openssl/src/openssl-VERSION/
8  (3) compiles OpenSSL
9  (4) installs OpenSSL into ../openssl/VERSION/
10  (5) forces a recompilation of Python modules using the
11      header and library files from ../openssl/VERSION/
12  (6) runs Python's test suite
13
14The script must be run with Python's build directory as current working
15directory:
16
17    ./python Tools/ssl/test_multiple_versions.py
18
19The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend
20search paths for header files and shared libraries. It's known to work on
21Linux with GCC 4.x.
22
23(c) 2013 Christian Heimes <christian@python.org>
24"""
25import logging
26import os
27import tarfile
28import shutil
29import subprocess
30import sys
31from urllib import urlopen
32
33log = logging.getLogger("multissl")
34
35OPENSSL_VERSIONS = [
36    "0.9.7m", "0.9.8i", "0.9.8l", "0.9.8m", "0.9.8y", "1.0.0k", "1.0.1e"
37]
38FULL_TESTS = [
39    "test_asyncio", "test_ftplib", "test_hashlib", "test_httplib",
40    "test_imaplib", "test_nntplib", "test_poplib", "test_smtplib",
41    "test_smtpnet", "test_urllib2_localnet", "test_venv"
42]
43MINIMAL_TESTS = ["test_ssl", "test_hashlib"]
44CADEFAULT = True
45HERE = os.path.abspath(os.getcwd())
46DEST_DIR = os.path.abspath(os.path.join(HERE, os.pardir, "openssl"))
47
48
49class BuildSSL(object):
50    url_template = "https://www.openssl.org/source/openssl-{}.tar.gz"
51
52    module_files = ["Modules/_ssl.c",
53                    "Modules/socketmodule.c",
54                    "Modules/_hashopenssl.c"]
55
56    def __init__(self, version, openssl_compile_args=(), destdir=DEST_DIR):
57        self._check_python_builddir()
58        self.version = version
59        self.openssl_compile_args = openssl_compile_args
60        # installation directory
61        self.install_dir = os.path.join(destdir, version)
62        # source file
63        self.src_file = os.path.join(destdir, "src",
64                                     "openssl-{}.tar.gz".format(version))
65        # build directory (removed after install)
66        self.build_dir = os.path.join(destdir, "src",
67                                      "openssl-{}".format(version))
68
69    @property
70    def openssl_cli(self):
71        """openssl CLI binary"""
72        return os.path.join(self.install_dir, "bin", "openssl")
73
74    @property
75    def openssl_version(self):
76        """output of 'bin/openssl version'"""
77        env = os.environ.copy()
78        env["LD_LIBRARY_PATH"] = self.lib_dir
79        cmd = [self.openssl_cli, "version"]
80        return self._subprocess_output(cmd, env=env)
81
82    @property
83    def pyssl_version(self):
84        """Value of ssl.OPENSSL_VERSION"""
85        env = os.environ.copy()
86        env["LD_LIBRARY_PATH"] = self.lib_dir
87        cmd = ["./python", "-c", "import ssl; print(ssl.OPENSSL_VERSION)"]
88        return self._subprocess_output(cmd, env=env)
89
90    @property
91    def include_dir(self):
92        return os.path.join(self.install_dir, "include")
93
94    @property
95    def lib_dir(self):
96        return os.path.join(self.install_dir, "lib")
97
98    @property
99    def has_openssl(self):
100        return os.path.isfile(self.openssl_cli)
101
102    @property
103    def has_src(self):
104        return os.path.isfile(self.src_file)
105
106    def _subprocess_call(self, cmd, stdout=subprocess.DEVNULL, env=None,
107                         **kwargs):
108        log.debug("Call '{}'".format(" ".join(cmd)))
109        return subprocess.check_call(cmd, stdout=stdout, env=env, **kwargs)
110
111    def _subprocess_output(self, cmd, env=None, **kwargs):
112        log.debug("Call '{}'".format(" ".join(cmd)))
113        out = subprocess.check_output(cmd, env=env)
114        return out.strip().decode("utf-8")
115
116    def _check_python_builddir(self):
117        if not os.path.isfile("python") or not os.path.isfile("setup.py"):
118            raise ValueError("Script must be run in Python build directory")
119
120    def _download_openssl(self):
121        """Download OpenSSL source dist"""
122        src_dir = os.path.dirname(self.src_file)
123        if not os.path.isdir(src_dir):
124            os.makedirs(src_dir)
125        url = self.url_template.format(self.version)
126        log.info("Downloading OpenSSL from {}".format(url))
127        req = urlopen(url, cadefault=CADEFAULT)
128        # KISS, read all, write all
129        data = req.read()
130        log.info("Storing {}".format(self.src_file))
131        with open(self.src_file, "wb") as f:
132            f.write(data)
133
134    def _unpack_openssl(self):
135        """Unpack tar.gz bundle"""
136        # cleanup
137        if os.path.isdir(self.build_dir):
138            shutil.rmtree(self.build_dir)
139        os.makedirs(self.build_dir)
140
141        tf = tarfile.open(self.src_file)
142        base = "openssl-{}/".format(self.version)
143        # force extraction into build dir
144        members = tf.getmembers()
145        for member in members:
146            if not member.name.startswith(base):
147                raise ValueError(member.name)
148            member.name = member.name[len(base):]
149        log.info("Unpacking files to {}".format(self.build_dir))
150        tf.extractall(self.build_dir, members)
151
152    def _build_openssl(self):
153        """Now build openssl"""
154        log.info("Running build in {}".format(self.install_dir))
155        cwd = self.build_dir
156        cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
157        cmd.extend(self.openssl_compile_args)
158        self._subprocess_call(cmd, cwd=cwd)
159        self._subprocess_call(["make"], cwd=cwd)
160
161    def _install_openssl(self, remove=True):
162        self._subprocess_call(["make", "install"], cwd=self.build_dir)
163        if remove:
164            shutil.rmtree(self.build_dir)
165
166    def install_openssl(self):
167        if not self.has_openssl:
168            if not self.has_src:
169                self._download_openssl()
170            else:
171                log.debug("Already has src {}".format(self.src_file))
172            self._unpack_openssl()
173            self._build_openssl()
174            self._install_openssl()
175        else:
176            log.info("Already has installation {}".format(self.install_dir))
177        # validate installation
178        version = self.openssl_version
179        if self.version not in version:
180            raise ValueError(version)
181
182    def touch_pymods(self):
183        # force a rebuild of all modules that use OpenSSL APIs
184        for fname in self.module_files:
185            os.utime(fname)
186
187    def recompile_pymods(self):
188        log.info("Using OpenSSL build from {}".format(self.build_dir))
189        # overwrite header and library search paths
190        env = os.environ.copy()
191        env["CPPFLAGS"] = "-I{}".format(self.include_dir)
192        env["LDFLAGS"] = "-L{}".format(self.lib_dir)
193        # set rpath
194        env["LD_RUN_PATH"] = self.lib_dir
195
196        log.info("Rebuilding Python modules")
197        self.touch_pymods()
198        cmd = ["./python", "setup.py", "build"]
199        self._subprocess_call(cmd, env=env)
200
201    def check_pyssl(self):
202        version = self.pyssl_version
203        if self.version not in version:
204            raise ValueError(version)
205
206    def run_pytests(self, *args):
207        cmd = ["./python", "-m", "test"]
208        cmd.extend(args)
209        self._subprocess_call(cmd, stdout=None)
210
211    def run_python_tests(self, *args):
212        self.recompile_pymods()
213        self.check_pyssl()
214        self.run_pytests(*args)
215
216
217def main(*args):
218    builders = []
219    for version in OPENSSL_VERSIONS:
220        if version in ("0.9.8i", "0.9.8l"):
221            openssl_compile_args = ("no-asm",)
222        else:
223            openssl_compile_args = ()
224        builder = BuildSSL(version, openssl_compile_args)
225        builder.install_openssl()
226        builders.append(builder)
227
228    for builder in builders:
229        builder.run_python_tests(*args)
230    # final touch
231    builder.touch_pymods()
232
233
234if __name__ == "__main__":
235    logging.basicConfig(level=logging.INFO,
236                        format="*** %(levelname)s %(message)s")
237    args = sys.argv[1:]
238    if not args:
239        args = ["-unetwork", "-v"]
240        args.extend(FULL_TESTS)
241    main(*args)
242