122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian#!/usr/bin/python 222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian#pylint: disable-msg=C0111 322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian# Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian# Use of this source code is governed by a BSD-style license that can be 622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian# found in the LICENSE file. 722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianimport common 922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 1022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.client.common_lib import global_config 1122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.client.common_lib.test_utils import unittest 1222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.frontend import setup_django_environment 1322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.frontend.afe import frontend_test_utils 1422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.frontend.afe import models 1522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.scheduler import rdb_testing_utils 1622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.scheduler import scheduler_models 1722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianfrom autotest_lib.scheduler.shard import shard_client 1822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 1922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 2022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanianclass ShardClientIntegrationTest(rdb_testing_utils.AbstractBaseRDBTester, 2122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian unittest.TestCase): 2222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Integration tests for the shard_client.""" 2322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 2422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 2522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def setup_global_config(self): 2622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Mock out global_config for shard client creation.""" 2722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian global_config.global_config.override_config_value( 2822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 'SHARD', 'is_slave_shard', 'True') 2922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian global_config.global_config.override_config_value( 3022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 'SHARD', 'shard_hostname', 'host1') 3122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 3222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 33af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian def initialize_shard_client(self): 34af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian self.setup_global_config() 35af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian return shard_client.get_shard_client() 36af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 37af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 3822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def testCompleteStatusBasic(self): 3922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Test that complete jobs are uploaded properly.""" 4022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 41af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian client = self.initialize_shard_client() 4222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job = self.create_job(deps=set(['a']), shard_hostname=client.hostname) 4322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian scheduler_models.initialize() 4422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian hqe = scheduler_models.HostQueueEntry.fetch( 4522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian where='job_id = %s' % job.id)[0] 4622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 4722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # This should set both the shard_id and the complete bit. 4822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian hqe.set_status('Completed') 4922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 5022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # Only incomplete jobs should be in known ids. 5122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job_ids, host_ids = client._get_known_ids() 5222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian assert(job_ids == []) 5322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 5422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # Jobs that have successfully gone through a set_status should 5522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # be ready for upload. 5622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian jobs = client._get_jobs_to_upload() 5722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian assert(job.id in [j.id for j in jobs]) 5822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 5922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 6022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def testOnlyShardId(self): 6122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Test that setting only the shardid prevents the job from upload.""" 6222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 63af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian client = self.initialize_shard_client() 6422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job = self.create_job(deps=set(['a']), shard_hostname=client.hostname) 6522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian scheduler_models.initialize() 6622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian hqe = scheduler_models.HostQueueEntry.fetch( 6722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian where='job_id = %s' % job.id)[0] 6822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 6922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian def _local_update_field(hqe, field_name, value): 7022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian """Turns update_field on the complete field into a no-op.""" 7122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian if field_name == 'complete': 7222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian return 7322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian models.HostQueueEntry.objects.filter(id=hqe.id).update( 7422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian **{field_name: value}) 7522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian setattr(hqe, field_name, value) 7622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 7722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian self.god.stub_with(scheduler_models.HostQueueEntry, 'update_field', 7822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian _local_update_field) 7922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 8022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # This should only update the shard_id. 8122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian hqe.set_status('Completed') 8222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 8322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # Retrieve the hqe along an independent code path so we're assured of 8422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # freshness, then make sure it has shard=None and an unset complete bit. 8522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian modified_hqe = self.db_helper.get_hqes(job_id=job.id)[0] 8622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian assert(modified_hqe.id == hqe.id and 8722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian modified_hqe.complete == 0 and 8822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian modified_hqe.job.shard == None) 8922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 9022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # Make sure the job with a shard but without complete is still 9122dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # in known_ids. 9222dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian job_ids, host_ids = client._get_known_ids() 9322dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian assert(set(job_ids) == set([job.id])) 9422dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 9522dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # Make sure the job with a shard but without complete is not 9622dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian # in uploaded jobs. 9722dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian jobs = client._get_jobs_to_upload() 9822dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian assert(jobs == []) 9922dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 10022dd226625255110c079e979113dcda1f4fa5ea8Prashanth Balasubramanian 101af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian def testHostSerialization(self): 102af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian """Test simple host serialization.""" 103af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian client = self.initialize_shard_client() 104af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian host = self.db_helper.create_host(name='test_host') 105af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian serialized_host = host.serialize() 106af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.Host.objects.all().delete() 107af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.Host.deserialize(serialized_host) 108af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.Host.objects.get(hostname='test_host') 109af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 110af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 111af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian def testUserExists(self): 112af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian """Test user related race conditions.""" 113af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian client = self.initialize_shard_client() 114af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian user = self.db_helper.create_user(name='test_user') 115af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian serialized_user = user.serialize() 116af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 117af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # Master sends a user with the same login but different id 118af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian serialized_user['id'] = '3' 119af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.deserialize(serialized_user) 120af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.objects.get(id=3, login='test_user') 121af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 122af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # Master sends a user with the same id, different login 123af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian serialized_user['login'] = 'fake_user' 124af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.deserialize(serialized_user) 125af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.objects.get(id=3, login='fake_user') 126af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 127af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian # Master sends a new user 128af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian user = self.db_helper.create_user(name='new_user') 129af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian serialized_user = user.serialize() 130af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.objects.all().delete() 131af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.deserialize(serialized_user) 132af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian models.User.objects.get(login='new_user') 133af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 134af5166418be945356238768cfc8226f3c2aa7138Prashanth Balasubramanian 135