1# Copyright (C) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import time
30import logging
31import re
32import urllib
33import webapp2
34
35from google.appengine.api import users
36from google.appengine.ext.webapp import template
37from google.appengine.ext import db
38
39from model.jsonresults import JsonResults
40from model.testfile import TestFile
41
42PARAM_MASTER = "master"
43PARAM_BUILDER = "builder"
44PARAM_DIR = "dir"
45PARAM_FILE = "file"
46PARAM_NAME = "name"
47PARAM_BEFORE = "before"
48PARAM_NUM_FILES = "numfiles"
49PARAM_KEY = "key"
50PARAM_TEST_TYPE = "testtype"
51PARAM_TEST_LIST_JSON = "testlistjson"
52PARAM_CALLBACK = "callback"
53
54
55def _replace_jsonp_callback(json, callback_name):
56    if callback_name and re.search(r"^[A-Za-z0-9_]+$", callback_name):
57        if re.search(r"^[A-Za-z0-9_]+[(]", json):
58            return re.sub(r"^[A-Za-z0-9_]+[(]", callback_name + "(", json)
59        return callback_name + "(" + json + ")"
60
61    return json
62
63
64class DeleteFile(webapp2.RequestHandler):
65    """Delete test file for a given builder and name from datastore."""
66
67    def get(self):
68        key = self.request.get(PARAM_KEY)
69        master = self.request.get(PARAM_MASTER)
70        builder = self.request.get(PARAM_BUILDER)
71        test_type = self.request.get(PARAM_TEST_TYPE)
72        name = self.request.get(PARAM_NAME)
73        num_files = self.request.get(PARAM_NUM_FILES)
74        before = self.request.get(PARAM_BEFORE)
75
76        logging.debug(
77            "Deleting File, master: %s, builder: %s, test_type: %s, name: %s, before: %s, key: %s.",
78            master, builder, test_type, name, before, key)
79
80        limit = int(num_files) if num_files else 1
81        num_deleted = TestFile.delete_file(key, master, builder, test_type, name, before, limit)
82
83        self.response.set_status(200)
84        self.response.out.write("Deleted %d files." % num_deleted)
85
86
87class GetFile(webapp2.RequestHandler):
88    """Get file content or list of files for given builder and name."""
89
90    def _get_file_list(self, master, builder, test_type, name, before, limit, callback_name=None):
91        """Get and display a list of files that matches builder and file name.
92
93        Args:
94            builder: builder name
95            test_type: type of the test
96            name: file name
97        """
98
99        files = TestFile.get_files(
100            master, builder, test_type, name, before, load_data=False, limit=limit)
101        if not files:
102            logging.info("File not found, master: %s, builder: %s, test_type: %s, name: %s.",
103                         master, builder, test_type, name)
104            self.response.out.write("File not found")
105            return
106
107        template_values = {
108            "admin": users.is_current_user_admin(),
109            "master": master,
110            "builder": builder,
111            "test_type": test_type,
112            "name": name,
113            "files": files,
114        }
115        if callback_name:
116            json = template.render("templates/showfilelist.jsonp", template_values)
117            self._serve_json(_replace_jsonp_callback(json, callback_name), files[0].date)
118            return
119        self.response.out.write(template.render("templates/showfilelist.html",
120                                                template_values))
121
122    def _get_file_content(self, master, builder, test_type, name):
123        """Return content of the file that matches builder and file name.
124
125        Args:
126            builder: builder name
127            test_type: type of the test
128            name: file name
129        """
130
131        files = TestFile.get_files(
132            master, builder, test_type, name, load_data=True, limit=1)
133        if not files:
134            logging.info("File not found, master %s, builder: %s, test_type: %s, name: %s.",
135                         master, builder, test_type, name)
136            return None, None
137
138        return files[0].data, files[0].date
139
140    def _get_file_content_from_key(self, key):
141        file = db.get(key)
142
143        if not file:
144            logging.info("File not found, key %s.", key)
145            return None
146
147        file.load_data()
148        return file.data, file.date
149
150    def _get_test_list_json(self, master, builder, test_type):
151        """Return json file with test name list only, do not include test
152           results and other non-test-data .
153
154        Args:
155            builder: builder name.
156            test_type: type of test results.
157        """
158
159        json, date = self._get_file_content(master, builder, test_type, "results.json")
160        if not json:
161            return None
162
163        return JsonResults.get_test_list(builder, json), date
164
165    def _serve_json(self, json, modified_date):
166        if json:
167            if "If-Modified-Since" in self.request.headers:
168                old_date = self.request.headers["If-Modified-Since"]
169                if time.strptime(old_date, '%a, %d %b %Y %H:%M:%S %Z') == modified_date.utctimetuple():
170                    self.response.set_status(304)
171                    return
172
173            # The appengine datetime objects are naive, so they lack a timezone.
174            # In practice, appengine seems to use GMT.
175            self.response.headers["Last-Modified"] = modified_date.strftime('%a, %d %b %Y %H:%M:%S') + ' GMT'
176            self.response.headers["Content-Type"] = "application/json"
177            self.response.headers["Access-Control-Allow-Origin"] = "*"
178            self.response.out.write(json)
179        else:
180            self.error(404)
181
182    def get(self):
183        key = self.request.get(PARAM_KEY)
184        master = self.request.get(PARAM_MASTER)
185        builder = self.request.get(PARAM_BUILDER)
186        test_type = self.request.get(PARAM_TEST_TYPE)
187        name = self.request.get(PARAM_NAME)
188        before = self.request.get(PARAM_BEFORE)
189        num_files = self.request.get(PARAM_NUM_FILES)
190        test_list_json = self.request.get(PARAM_TEST_LIST_JSON)
191        callback_name = self.request.get(PARAM_CALLBACK)
192
193        logging.debug(
194            "Getting files, master %s, builder: %s, test_type: %s, name: %s, before: %s.",
195            master, builder, test_type, name, before)
196
197        if key:
198            json, date = self._get_file_content_from_key(key)
199        elif test_list_json:
200            json, date = self._get_test_list_json(master, builder, test_type)
201        elif num_files or not master or not builder or not test_type or not name:
202            limit = int(num_files) if num_files else 100
203            self._get_file_list(master, builder, test_type, name, before, limit, callback_name)
204            return
205        else:
206            json, date = self._get_file_content(master, builder, test_type, name)
207
208        if json:
209            json = _replace_jsonp_callback(json, callback_name)
210
211        self._serve_json(json, date)
212
213
214class Upload(webapp2.RequestHandler):
215    """Upload test results file to datastore."""
216
217    def post(self):
218        file_params = self.request.POST.getall(PARAM_FILE)
219        if not file_params:
220            self.response.out.write("FAIL: missing upload file field.")
221            return
222
223        builder = self.request.get(PARAM_BUILDER)
224        if not builder:
225            self.response.out.write("FAIL: missing builder parameter.")
226            return
227
228        master = self.request.get(PARAM_MASTER)
229        test_type = self.request.get(PARAM_TEST_TYPE)
230
231        logging.debug(
232            "Processing upload request, master: %s, builder: %s, test_type: %s.",
233            master, builder, test_type)
234
235        # There are two possible types of each file_params in the request:
236        # one file item or a list of file items.
237        # Normalize file_params to a file item list.
238        files = []
239        logging.debug("test: %s, type:%s", file_params, type(file_params))
240        for item in file_params:
241            if not isinstance(item, list) and not isinstance(item, tuple):
242                item = [item]
243            files.extend(item)
244
245        errors = []
246        final_status_code = 200
247        for file in files:
248            if file.filename == "incremental_results.json":
249                status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=False)
250            elif file.filename == "times_ms.json":
251                # We never look at historical times_ms.json files, so we can overwrite the existing one if it exists.
252                status_string, status_code = TestFile.overwrite_or_add_file(master, builder, test_type, file.filename, file.value)
253            else:
254                status_string, status_code = TestFile.add_file(master, builder, test_type, file.filename, file.value)
255                # FIXME: Upload full_results.json files for non-layout tests as well and stop supporting the
256                # incremental_results.json file format.
257                if status_code == 200 and file.filename == "full_results.json":
258                    status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=True)
259
260            if status_code == 200:
261                logging.info(status_string)
262            else:
263                logging.error(status_string)
264                errors.append(status_string)
265                final_status_code = status_code
266
267        if errors:
268            messages = "FAIL: " + "; ".join(errors)
269            self.response.set_status(final_status_code, messages)
270            self.response.out.write(messages)
271        else:
272            self.response.set_status(200)
273            self.response.out.write("OK")
274
275
276class UploadForm(webapp2.RequestHandler):
277    """Show a form so user can upload a file."""
278
279    def get(self):
280        template_values = {
281            "upload_url": "/testfile/upload",
282        }
283        self.response.out.write(template.render("templates/uploadform.html",
284                                                template_values))
285