1import logging, os, re
2from autotest_lib.client.common_lib import error
3from autotest_lib.client.virt import virt_utils, rss_client, aexpect
4
5
6def run_whql_submission(test, params, env):
7    """
8    WHQL submission test:
9    1) Log into the client machines and into a DTM server machine
10    2) Copy the automation program binary (dsso_test_binary) to the server machine
11    3) Run the automation program
12    4) Pass the program all relevant parameters (e.g. device_data)
13    5) Wait for the program to terminate
14    6) Parse and report job results
15    (logs and HTML reports are placed in test.debugdir)
16
17    @param test: kvm test object
18    @param params: Dictionary with the test parameters
19    @param env: Dictionary with test environment.
20    """
21    # Log into all client VMs
22    login_timeout = int(params.get("login_timeout", 360))
23    vms = []
24    sessions = []
25    for vm_name in params.objects("vms"):
26        vms.append(env.get_vm(vm_name))
27        vms[-1].verify_alive()
28        sessions.append(vms[-1].wait_for_login(timeout=login_timeout))
29
30    # Make sure all NICs of all client VMs are up
31    for vm in vms:
32        nics = vm.params.objects("nics")
33        for nic_index in range(len(nics)):
34            s = vm.wait_for_login(nic_index, 600)
35            s.close()
36
37    # Collect parameters
38    server_address = params.get("server_address")
39    server_shell_port = int(params.get("server_shell_port"))
40    server_file_transfer_port = int(params.get("server_file_transfer_port"))
41    server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
42                                    "Microsoft Driver Test Manager\\Studio")
43    dsso_test_binary = params.get("dsso_test_binary",
44                                  "deps/whql_submission_15.exe")
45    dsso_test_binary = virt_utils.get_path(test.bindir, dsso_test_binary)
46    dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
47                                            "deps/whql_delete_machine_15.exe")
48    dsso_delete_machine_binary = virt_utils.get_path(test.bindir,
49                                                    dsso_delete_machine_binary)
50    test_timeout = float(params.get("test_timeout", 600))
51
52    # Copy dsso binaries to the server
53    for filename in dsso_test_binary, dsso_delete_machine_binary:
54        rss_client.upload(server_address, server_file_transfer_port,
55                                 filename, server_studio_path, timeout=60)
56
57    # Open a shell session with the server
58    server_session = virt_utils.remote_login("nc", server_address,
59                                            server_shell_port, "", "",
60                                            sessions[0].prompt,
61                                            sessions[0].linesep)
62    server_session.set_status_test_command(sessions[0].status_test_command)
63
64    # Get the computer names of the server and clients
65    cmd = "echo %computername%"
66    server_name = server_session.cmd_output(cmd).strip()
67    client_names = [session.cmd_output(cmd).strip() for session in sessions]
68
69    # Delete all client machines from the server's data store
70    server_session.cmd("cd %s" % server_studio_path)
71    for client_name in client_names:
72        cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
73                            server_name, client_name)
74        server_session.cmd(cmd, print_func=logging.debug)
75
76    # Reboot the client machines
77    sessions = virt_utils.parallel((vm.reboot, (session,))
78                                  for vm, session in zip(vms, sessions))
79
80    # Check the NICs again
81    for vm in vms:
82        nics = vm.params.objects("nics")
83        for nic_index in range(len(nics)):
84            s = vm.wait_for_login(nic_index, 600)
85            s.close()
86
87    # Run whql_pre_command and close the sessions
88    if params.get("whql_pre_command"):
89        for session in sessions:
90            session.cmd(params.get("whql_pre_command"),
91                        int(params.get("whql_pre_command_timeout", 600)))
92            session.close()
93
94    # Run the automation program on the server
95    pool_name = "%s_pool" % client_names[0]
96    submission_name = "%s_%s" % (client_names[0],
97                                 params.get("submission_name"))
98    cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
99                                 server_name, pool_name, submission_name,
100                                 test_timeout, " ".join(client_names))
101    server_session.sendline(cmd)
102
103    # Helper function: wait for a given prompt and raise an exception if an
104    # error occurs
105    def find_prompt(prompt):
106        m, o = server_session.read_until_last_line_matches(
107            [prompt, server_session.prompt], print_func=logging.info,
108            timeout=600)
109        if m != 0:
110            errors = re.findall("^Error:.*$", o, re.I | re.M)
111            if errors:
112                raise error.TestError(errors[0])
113            else:
114                raise error.TestError("Error running automation program: "
115                                      "could not find '%s' prompt" % prompt)
116
117    # Tell the automation program which device to test
118    find_prompt("Device to test:")
119    server_session.sendline(params.get("test_device"))
120
121    # Tell the automation program which jobs to run
122    find_prompt("Jobs to run:")
123    server_session.sendline(params.get("job_filter", ".*"))
124
125    # Set submission DeviceData
126    find_prompt("DeviceData name:")
127    for dd in params.objects("device_data"):
128        dd_params = params.object_params(dd)
129        if dd_params.get("dd_name") and dd_params.get("dd_data"):
130            server_session.sendline(dd_params.get("dd_name"))
131            server_session.sendline(dd_params.get("dd_data"))
132    server_session.sendline()
133
134    # Set submission descriptors
135    find_prompt("Descriptor path:")
136    for desc in params.objects("descriptors"):
137        desc_params = params.object_params(desc)
138        if desc_params.get("desc_path"):
139            server_session.sendline(desc_params.get("desc_path"))
140    server_session.sendline()
141
142    # Set machine dimensions for each client machine
143    for vm_name in params.objects("vms"):
144        vm_params = params.object_params(vm_name)
145        find_prompt(r"Dimension name\b.*:")
146        for dp in vm_params.objects("dimensions"):
147            dp_params = vm_params.object_params(dp)
148            if dp_params.get("dim_name") and dp_params.get("dim_value"):
149                server_session.sendline(dp_params.get("dim_name"))
150                server_session.sendline(dp_params.get("dim_value"))
151        server_session.sendline()
152
153    # Set extra parameters for tests that require them (e.g. NDISTest)
154    for vm_name in params.objects("vms"):
155        vm_params = params.object_params(vm_name)
156        find_prompt(r"Parameter name\b.*:")
157        for dp in vm_params.objects("device_params"):
158            dp_params = vm_params.object_params(dp)
159            if dp_params.get("dp_name") and dp_params.get("dp_regex"):
160                server_session.sendline(dp_params.get("dp_name"))
161                server_session.sendline(dp_params.get("dp_regex"))
162                # Make sure the prompt appears again (if the device isn't found
163                # the automation program will terminate)
164                find_prompt(r"Parameter name\b.*:")
165        server_session.sendline()
166
167    # Wait for the automation program to terminate
168    try:
169        o = server_session.read_up_to_prompt(print_func=logging.info,
170                                             timeout=test_timeout + 300)
171        # (test_timeout + 300 is used here because the automation program is
172        # supposed to terminate cleanly on its own when test_timeout expires)
173        done = True
174    except aexpect.ExpectError, e:
175        o = e.output
176        done = False
177    server_session.close()
178
179    # Look for test results in the automation program's output
180    result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL)
181    if not result_summaries:
182        raise error.TestError("The automation program did not return any "
183                              "results")
184    results = result_summaries[-1].strip("-")
185    results = eval("".join(results.splitlines()))
186
187    # Download logs and HTML reports from the server
188    for i, r in enumerate(results):
189        if "report" in r:
190            try:
191                rss_client.download(server_address,
192                                           server_file_transfer_port,
193                                           r["report"], test.debugdir)
194            except rss_client.FileTransferNotFoundError:
195                pass
196        if "logs" in r:
197            try:
198                rss_client.download(server_address,
199                                           server_file_transfer_port,
200                                           r["logs"], test.debugdir)
201            except rss_client.FileTransferNotFoundError:
202                pass
203            else:
204                try:
205                    # Create symlinks to test log dirs to make it easier
206                    # to access them (their original names are not human
207                    # readable)
208                    link_name = "logs_%s" % r["report"].split("\\")[-1]
209                    link_name = link_name.replace(" ", "_")
210                    link_name = link_name.replace("/", "_")
211                    os.symlink(r["logs"].split("\\")[-1],
212                               os.path.join(test.debugdir, link_name))
213                except (KeyError, OSError):
214                    pass
215
216    # Print result summary (both to the regular logs and to a file named
217    # 'summary' in test.debugdir)
218    def print_summary_line(f, line):
219        logging.info(line)
220        f.write(line + "\n")
221    if results:
222        # Make sure all results have the required keys
223        for r in results:
224            r["id"] = str(r.get("id"))
225            r["job"] = str(r.get("job"))
226            r["status"] = str(r.get("status"))
227            r["pass"] = int(r.get("pass", 0))
228            r["fail"] = int(r.get("fail", 0))
229            r["notrun"] = int(r.get("notrun", 0))
230            r["notapplicable"] = int(r.get("notapplicable", 0))
231        # Sort the results by failures and total test count in descending order
232        results = [(r["fail"],
233                    r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"],
234                    r) for r in results]
235        results.sort(reverse=True)
236        results = [r[-1] for r in results]
237        # Print results
238        logging.info("")
239        logging.info("Result summary:")
240        name_length = max(len(r["job"]) for r in results)
241        fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
242        f = open(os.path.join(test.debugdir, "summary"), "w")
243        print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
244                                     "NotRun", "NotApplicable"))
245        print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
246                                     "------", "-------------"))
247        for r in results:
248            print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
249                                         r["pass"], r["fail"], r["notrun"],
250                                         r["notapplicable"]))
251        f.close()
252        logging.info("(see logs and HTML reports in %s)", test.debugdir)
253
254    # Kill the client VMs and fail if the automation program did not terminate
255    # on time
256    if not done:
257        virt_utils.parallel(vm.destroy for vm in vms)
258        raise error.TestFail("The automation program did not terminate "
259                             "on time")
260
261    # Fail if there are failed or incomplete jobs (kill the client VMs if there
262    # are incomplete jobs)
263    failed_jobs = [r["job"] for r in results
264                   if r["status"].lower() == "investigate"]
265    running_jobs = [r["job"] for r in results
266                    if r["status"].lower() == "inprogress"]
267    errors = []
268    if failed_jobs:
269        errors += ["Jobs failed: %s." % failed_jobs]
270    if running_jobs:
271        for vm in vms:
272            vm.destroy()
273        errors += ["Jobs did not complete on time: %s." % running_jobs]
274    if errors:
275        raise error.TestFail(" ".join(errors))
276