15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2010 Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import time 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import logging 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import re 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import urllib 335267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)import webapp2 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from google.appengine.api import users 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from google.appengine.ext.webapp import template 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from google.appengine.ext import db 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from model.jsonresults import JsonResults 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from model.testfile import TestFile 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_MASTER = "master" 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_BUILDER = "builder" 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_DIR = "dir" 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_FILE = "file" 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_NAME = "name" 475267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)PARAM_BEFORE = "before" 485267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)PARAM_NUM_FILES = "numfiles" 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_KEY = "key" 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_TEST_TYPE = "testtype" 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_TEST_LIST_JSON = "testlistjson" 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PARAM_CALLBACK = "callback" 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def _replace_jsonp_callback(json, callback_name): 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if callback_name and re.search(r"^[A-Za-z0-9_]+$", callback_name): 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if re.search(r"^[A-Za-z0-9_]+[(]", json): 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return re.sub(r"^[A-Za-z0-9_]+[(]", callback_name + "(", json) 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return callback_name + "(" + json + ")" 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return json 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 645267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)class DeleteFile(webapp2.RequestHandler): 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Delete test file for a given builder and name from datastore.""" 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def get(self): 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) key = self.request.get(PARAM_KEY) 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master = self.request.get(PARAM_MASTER) 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builder = self.request.get(PARAM_BUILDER) 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_type = self.request.get(PARAM_TEST_TYPE) 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) name = self.request.get(PARAM_NAME) 735267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) num_files = self.request.get(PARAM_NUM_FILES) 745267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) before = self.request.get(PARAM_BEFORE) 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.debug( 775267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) "Deleting File, master: %s, builder: %s, test_type: %s, name: %s, before: %s, key: %s.", 785267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) master, builder, test_type, name, before, key) 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 805267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) limit = int(num_files) if num_files else 1 815267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) num_deleted = TestFile.delete_file(key, master, builder, test_type, name, before, limit) 825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 835267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) self.response.set_status(200) 845267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) self.response.out.write("Deleted %d files." % num_deleted) 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 875267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)class GetFile(webapp2.RequestHandler): 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Get file content or list of files for given builder and name.""" 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 905267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) def _get_file_list(self, master, builder, test_type, name, before, limit, callback_name=None): 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Get and display a list of files that matches builder and file name. 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Args: 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builder: builder name 955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_type: type of the test 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) name: file name 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """ 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) files = TestFile.get_files( 1005267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) master, builder, test_type, name, before, load_data=False, limit=limit) 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not files: 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.info("File not found, master: %s, builder: %s, test_type: %s, name: %s.", 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master, builder, test_type, name) 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write("File not found") 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template_values = { 1085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "admin": users.is_current_user_admin(), 1095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "master": master, 1105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "builder": builder, 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "test_type": test_type, 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "name": name, 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "files": files, 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if callback_name: 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) json = template.render("templates/showfilelist.jsonp", template_values) 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._serve_json(_replace_jsonp_callback(json, callback_name), files[0].date) 1185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write(template.render("templates/showfilelist.html", 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template_values)) 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _get_file_content(self, master, builder, test_type, name): 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Return content of the file that matches builder and file name. 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Args: 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builder: builder name 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_type: type of the test 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) name: file name 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """ 1305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) files = TestFile.get_files( 1325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master, builder, test_type, name, load_data=True, limit=1) 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not files: 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.info("File not found, master %s, builder: %s, test_type: %s, name: %s.", 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master, builder, test_type, name) 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None, None 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return files[0].data, files[0].date 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _get_file_content_from_key(self, key): 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) file = db.get(key) 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not file: 1445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.info("File not found, key %s.", key) 1455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 1465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) file.load_data() 1485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return file.data, file.date 1495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _get_test_list_json(self, master, builder, test_type): 1515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Return json file with test name list only, do not include test 1525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) results and other non-test-data . 1535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Args: 1555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builder: builder name. 1565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_type: type of test results. 1575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """ 1585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) json, date = self._get_file_content(master, builder, test_type, "results.json") 1605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not json: 1615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 1625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return JsonResults.get_test_list(builder, json), date 1645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _serve_json(self, json, modified_date): 1665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if json: 1675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if "If-Modified-Since" in self.request.headers: 1685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) old_date = self.request.headers["If-Modified-Since"] 1695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if time.strptime(old_date, '%a, %d %b %Y %H:%M:%S %Z') == modified_date.utctimetuple(): 1705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.set_status(304) 1715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # The appengine datetime objects are naive, so they lack a timezone. 1745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # In practice, appengine seems to use GMT. 1755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.headers["Last-Modified"] = modified_date.strftime('%a, %d %b %Y %H:%M:%S') + ' GMT' 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.headers["Content-Type"] = "application/json" 1775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.headers["Access-Control-Allow-Origin"] = "*" 1785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write(json) 1795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 1805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.error(404) 1815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def get(self): 1835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) key = self.request.get(PARAM_KEY) 1845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master = self.request.get(PARAM_MASTER) 1855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builder = self.request.get(PARAM_BUILDER) 1865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_type = self.request.get(PARAM_TEST_TYPE) 1875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) name = self.request.get(PARAM_NAME) 1885267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) before = self.request.get(PARAM_BEFORE) 1895267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) num_files = self.request.get(PARAM_NUM_FILES) 1905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_list_json = self.request.get(PARAM_TEST_LIST_JSON) 1915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) callback_name = self.request.get(PARAM_CALLBACK) 1925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.debug( 1945267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) "Getting files, master %s, builder: %s, test_type: %s, name: %s, before: %s.", 1955267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) master, builder, test_type, name, before) 1965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if key: 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) json, date = self._get_file_content_from_key(key) 1995267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) elif test_list_json: 2005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) json, date = self._get_test_list_json(master, builder, test_type) 2015267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) elif num_files or not master or not builder or not test_type or not name: 2025267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) limit = int(num_files) if num_files else 100 2035267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) self._get_file_list(master, builder, test_type, name, before, limit, callback_name) 2045267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) return 2055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 2065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) json, date = self._get_file_content(master, builder, test_type, name) 2075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if json: 2095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) json = _replace_jsonp_callback(json, callback_name) 2105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._serve_json(json, date) 2125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2145267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)class Upload(webapp2.RequestHandler): 2155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Upload test results file to datastore.""" 2165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def post(self): 2185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) file_params = self.request.POST.getall(PARAM_FILE) 2195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not file_params: 2205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write("FAIL: missing upload file field.") 2215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builder = self.request.get(PARAM_BUILDER) 2245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not builder: 2255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write("FAIL: missing builder parameter.") 2265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 2275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master = self.request.get(PARAM_MASTER) 2295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_type = self.request.get(PARAM_TEST_TYPE) 2305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.debug( 2325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "Processing upload request, master: %s, builder: %s, test_type: %s.", 2335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) master, builder, test_type) 2345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # There are two possible types of each file_params in the request: 2365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # one file item or a list of file items. 2375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Normalize file_params to a file item list. 2385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) files = [] 2395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) logging.debug("test: %s, type:%s", file_params, type(file_params)) 2405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for item in file_params: 2415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not isinstance(item, list) and not isinstance(item, tuple): 2425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) item = [item] 2435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) files.extend(item) 2445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) errors = [] 2465267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) final_status_code = 200 2475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for file in files: 24893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if file.filename == "incremental_results.json": 2495267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=False) 2505267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) elif file.filename == "times_ms.json": 2515267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) # We never look at historical times_ms.json files, so we can overwrite the existing one if it exists. 2525267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) status_string, status_code = TestFile.overwrite_or_add_file(master, builder, test_type, file.filename, file.value) 2535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 2545267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) status_string, status_code = TestFile.add_file(master, builder, test_type, file.filename, file.value) 25593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) # FIXME: Upload full_results.json files for non-layout tests as well and stop supporting the 25693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) # incremental_results.json file format. 2575267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if status_code == 200 and file.filename == "full_results.json": 2585267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) status_string, status_code = JsonResults.update(master, builder, test_type, file.value, is_full_results_format=True) 2595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2605267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) if status_code == 200: 2615267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) logging.info(status_string) 2625267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) else: 2635267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) logging.error(status_string) 2645267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) errors.append(status_string) 2655267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) final_status_code = status_code 2665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if errors: 2685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) messages = "FAIL: " + "; ".join(errors) 2695267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) self.response.set_status(final_status_code, messages) 2705267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles) self.response.out.write(messages) 2715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 2725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.set_status(200) 2735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write("OK") 2745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2765267f701546148b83dfbe1d151cb184385bb5c22Torne (Richard Coles)class UploadForm(webapp2.RequestHandler): 2775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Show a form so user can upload a file.""" 2785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def get(self): 2805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template_values = { 2815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "upload_url": "/testfile/upload", 2825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 2835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.response.out.write(template.render("templates/uploadform.html", 2845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) template_values)) 285