Lines Matching refs:shard
224 # Make sure the label exists on the shard with the same id
226 # It is possible that the label is already in a shard because
229 # For example, we add a label L1 to a host in shard S1.
352 If this is called on the master, but the host is assigned to a shard, this
353 will call `modify_host_local` RPC to the responsible shard. This means if
356 When this is called on a shard, the shard just routes the RPC to the master
373 # between the master and a shard.
386 # In case the shard update fails mid-flight and the master-shard desync, we
388 # (automated) corrective actions will revert the (partial) shard updates.
406 filters is assigned to a shard, this will call `modify_hosts_local` RPC
407 to the responsible shard.
408 When this is called on a shard, the shard just routes the RPC to the master
412 if the states of a host differ on the master and a shard, the state on the
417 update the host (both on the shard and on the master), because the state
442 if host.shard:
443 affected_shard_hostnames.add(host.shard.hostname)
447 # between the master and a shard.
605 This RPC exists for master-shard communication only.
1131 For shard, create special task records in local DB.
1142 # Filter out hosts on a shard from those on the master, forward
1143 # rpcs to the shard with an additional hostname__in filter, and
1146 hosts = [h for h in hosts if h.shard is None]
1147 for shard, hostnames in shard_host_map.iteritems():
1152 # each shard even though we supply all the ids in filter_data.
1159 rpc, [shard], **shard_filter)
1161 # There is a race condition here if someone assigns a shard to one of these
1166 # to the shard as we're creating a task here.
1170 # a verify running on a Ready host while the shard is using it, if the
1444 The caller is expected to know the target shard for such reasons
1447 to operate on behalf of one specific shard, and no other.
1449 the shard assigned to that host.
1466 shard, forward this call to that shard.
1475 if not host.shard:
1481 # the shard.
1482 shard_afe = frontend.AFE(server=host.shard.hostname)
1506 shard, forward this call to that shard.
1514 if not host_model.shard:
1517 shard_afe = frontend.AFE(server=host_model.shard.hostname)
1522 """Get the "status task" for a host from the local shard.
1530 This call will not be forward to a shard; the receiving server
1531 must be the shard that owns the host.
1549 """Get the "status task" for a host from its owning shard.
1551 Finds the given host's owning shard, and forwards to it a call
1564 if not host.shard:
1570 # the shard.
1571 shard_afe = frontend.AFE(server=host.shard.hostname)
1603 if not host.shard or utils.is_shard():
1607 shard_afe = frontend.AFE(server=host.shard.hostname)
2150 @param shard_hostname: Hostname of the calling shard
2152 status from a shard.
2154 newer status from a shard. Note that for every hostqueueentry
2156 @param known_job_ids: List of ids of jobs the shard already has.
2157 @param known_host_ids: List of ids of hosts the shard already has.
2158 @param known_host_statuses: List of statuses of hosts the shard already has.
2161 and their dependencies to be inserted into a shard's database.
2167 # particular shard during a heartbeat, it never will be assigned to this
2168 # shard later.
2170 # to the shard now, but might be sent in a future heartbeat. This means
2172 # maximum host id the shard knows.
2173 # 2. Send the number of jobs/hosts the shard knows to the master in each
2175 # the shard_id set to this shard. In the normal case, they should match.
2218 @returns A sequence of nested dictionaries of shard information.
2222 for serialized, shard in zip(serialized_shards, shards):
2223 serialized['labels'] = [label.name for label in shard.labels.all()]
2229 """Verify whether board labels are valid to be added to a given shard.
2233 assigned to shard.
2238 or board has been added to shard already.
2241 @returns: A list of label models that ready to be added to shard.
2258 shard = models.Shard.objects.get(labels=label)
2260 '%s is already on shard %s' % (label, shard.hostname))
2262 # board is not on any shard, so it's valid.
2268 """Add a shard and start running jobs on it.
2270 @param hostname: The hostname of the shard to be added; needs to be unique.
2272 will be assigned to the shard.
2275 board has been added to shard already.
2276 @raises model_logic.ValidationError: If a shard with the given hostname
2280 @returns: The id of the added shard.
2283 shard = models.Shard.add_object(hostname=hostname)
2285 shard.labels.add(label)
2286 return shard.id
2290 """Add boards to a given shard
2292 @param hostname: The hostname of the shard to be changed.
2296 board has been added to shard already.
2299 @returns: The id of the changed shard.
2302 shard = models.Shard.objects.get(hostname=hostname)
2304 shard.labels.add(label)
2305 return shard.id
2315 """Remove board from the given shard.
2316 @param hostname: The hostname of the shard to be changed.
2321 @returns: The id of the changed shard.
2323 shard = models.Shard.objects.get(hostname=hostname)
2325 if label not in shard.labels.all():
2327 'Cannot remove label from shard that does not belong to it.')
2329 shard.labels.remove(label)
2333 static_labels__in=[static_label]).update(shard=None)
2335 models.Host.objects.filter(labels__in=[label]).update(shard=None)
2339 """Delete a shard and reclaim all resources from it.
2341 This claims back all assigned hosts from the shard.
2343 shard = rpc_utils.retrieve_shard(shard_hostname=hostname)
2345 # Remove shard information.
2346 models.Host.objects.filter(shard=shard).update(shard=None)
2349 # models.Job.objects.filter(shard=shard).update(shard=None)
2360 assert shard.id is not None
2362 res = cursor.execute(QUERY % shard.id).fetchone()
2368 models.Job.objects.filter(shard=shard).update(shard=None)
2372 shard.labels.clear()
2373 shard.delete()