chromiumsync.py revision ca12bfac764ba476d6cd062bf1dde12cc64c3f40
12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# found in the LICENSE file. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)"""An implementation of the server side of the Chromium sync protocol. 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The details of the protocol are described mostly by comments in the protocol 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)buffer definition at chrome/browser/sync/protocol/sync.proto. 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)""" 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import cgi 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import copy 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import operator 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import pickle 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import random 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import string 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sys 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import threading 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import time 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import urlparse 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import app_notification_specifics_pb2 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import app_setting_specifics_pb2 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import app_specifics_pb2 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import autofill_specifics_pb2 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import bookmark_specifics_pb2 272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import dictionary_specifics_pb2 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import get_updates_caller_info_pb2 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import extension_setting_specifics_pb2 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import extension_specifics_pb2 312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import favicon_image_specifics_pb2 322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import favicon_tracking_specifics_pb2 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import history_delete_directive_specifics_pb2 34c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import managed_user_setting_specifics_pb2 3590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)import managed_user_specifics_pb2 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import nigori_specifics_pb2 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import password_specifics_pb2 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import preference_specifics_pb2 39c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import priority_preference_specifics_pb2 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import search_engine_specifics_pb2 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import session_specifics_pb2 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sync_pb2 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sync_enums_pb2 442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import synced_notification_specifics_pb2 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import theme_specifics_pb2 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import typed_url_specifics_pb2 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# An enumeration of the various kinds of data that can be synced. 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Over the wire, this enumeration is not used: a sync object's type is 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# inferred by which EntitySpecifics field it has. But in the context 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# of a program, it is useful to have an enumeration. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ALL_TYPES = ( 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TOP_LEVEL, # The type of the 'Google Chrome' folder. 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APPS, 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_NOTIFICATION, 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_SETTINGS, 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL, 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL_PROFILE, 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BOOKMARK, 602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DEVICE_INFO, 612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DICTIONARY, 622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPERIMENTS, 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXTENSIONS, 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) HISTORY_DELETE_DIRECTIVE, 65c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) MANAGED_USER_SETTING, 6690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) MANAGED_USER, 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NIGORI, 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PASSWORD, 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PREFERENCE, 70c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PRIORITY_PREFERENCE, 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SEARCH_ENGINE, 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SESSION, 732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SYNCED_NOTIFICATION, 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) THEME, 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TYPED_URL, 762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXTENSION_SETTINGS, 772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) FAVICON_IMAGES, 7890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) FAVICON_TRACKING) = range(26) 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# An enumeration on the frequency at which the server should send errors 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# to the client. This would be specified by the url that triggers the error. 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Note: This enum should be kept in the same order as the enum in sync_test.h. 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SYNC_ERROR_FREQUENCY = ( 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_NONE, 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_ALWAYS, 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_TWO_THIRDS) = range(3) 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Well-known server tag of the top level 'Google Chrome' folder. 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TOP_LEVEL_FOLDER_TAG = 'google_chrome' 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Given a sync type from ALL_TYPES, find the FieldDescriptor corresponding 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# to that datatype. Note that TOP_LEVEL has no such token. 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SYNC_TYPE_FIELDS = sync_pb2.EntitySpecifics.DESCRIPTOR.fields_by_name 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SYNC_TYPE_TO_DESCRIPTOR = { 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_NOTIFICATION: SYNC_TYPE_FIELDS['app_notification'], 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_SETTINGS: SYNC_TYPE_FIELDS['app_setting'], 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APPS: SYNC_TYPE_FIELDS['app'], 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL: SYNC_TYPE_FIELDS['autofill'], 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL_PROFILE: SYNC_TYPE_FIELDS['autofill_profile'], 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BOOKMARK: SYNC_TYPE_FIELDS['bookmark'], 1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DEVICE_INFO: SYNC_TYPE_FIELDS['device_info'], 1022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DICTIONARY: SYNC_TYPE_FIELDS['dictionary'], 1032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPERIMENTS: SYNC_TYPE_FIELDS['experiments'], 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXTENSION_SETTINGS: SYNC_TYPE_FIELDS['extension_setting'], 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXTENSIONS: SYNC_TYPE_FIELDS['extension'], 1062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) FAVICON_IMAGES: SYNC_TYPE_FIELDS['favicon_image'], 1072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) FAVICON_TRACKING: SYNC_TYPE_FIELDS['favicon_tracking'], 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) HISTORY_DELETE_DIRECTIVE: SYNC_TYPE_FIELDS['history_delete_directive'], 109c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) MANAGED_USER_SETTING: SYNC_TYPE_FIELDS['managed_user_setting'], 11090dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) MANAGED_USER: SYNC_TYPE_FIELDS['managed_user'], 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NIGORI: SYNC_TYPE_FIELDS['nigori'], 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PASSWORD: SYNC_TYPE_FIELDS['password'], 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PREFERENCE: SYNC_TYPE_FIELDS['preference'], 114c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PRIORITY_PREFERENCE: SYNC_TYPE_FIELDS['priority_preference'], 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SEARCH_ENGINE: SYNC_TYPE_FIELDS['search_engine'], 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SESSION: SYNC_TYPE_FIELDS['session'], 1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SYNCED_NOTIFICATION: SYNC_TYPE_FIELDS["synced_notification"], 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) THEME: SYNC_TYPE_FIELDS['theme'], 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TYPED_URL: SYNC_TYPE_FIELDS['typed_url'], 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The parent ID used to indicate a top-level node. 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ROOT_ID = '0' 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Unix time epoch in struct_time format. The tuple corresponds to UTC Wednesday 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Jan 1 1970, 00:00:00, non-dst. 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)UNIX_TIME_EPOCH = (1970, 1, 1, 0, 0, 0, 3, 1, 0) 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The number of characters in the server-generated encryption key. 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)KEYSTORE_KEY_LENGTH = 16 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 132ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch# The hashed client tags for some experiment nodes. 1332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)KEYSTORE_ENCRYPTION_EXPERIMENT_TAG = "pis8ZRzh98/MKLtVEio2mr42LQA=" 134ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben MurdochPRE_COMMIT_GU_AVOIDANCE_EXPERIMENT_TAG = "Z1xgeh3QUBa50vdEPd8C/4c7jfE=" 1352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class Error(Exception): 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Error class for this module.""" 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ProtobufDataTypeFieldNotUnique(Error): 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """An entry should not have more than one data type present.""" 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class DataTypeIdNotRecognized(Error): 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The requested data type is not recognized.""" 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class MigrationDoneError(Error): 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A server-side migration occurred; clients must re-sync some datatypes. 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Attributes: 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatypes: a list of the datatypes (python enum) needing migration. 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, datatypes): 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.datatypes = datatypes 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class StoreBirthdayError(Error): 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The client sent a birthday that doesn't correspond to this server.""" 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class TransientError(Error): 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The client would be sent a transient error.""" 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class SyncInducedError(Error): 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The client would be sent an error.""" 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class InducedErrorFrequencyNotDefined(Error): 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The error frequency defined is not handled.""" 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetEntryType(entry): 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Extract the sync type from a SyncEntry. 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A SyncEntity protobuf object whose type to determine. 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) An enum value from ALL_TYPES if the entry's type can be determined, or None 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if the type cannot be determined. 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ProtobufDataTypeFieldNotUnique: More than one type was indicated by 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) the entry. 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.server_defined_unique_tag == TOP_LEVEL_FOLDER_TAG: 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return TOP_LEVEL 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry_types = GetEntryTypesFromSpecifics(entry.specifics) 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not entry_types: 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # If there is more than one, either there's a bug, or else the caller 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # should use GetEntryTypes. 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(entry_types) > 1: 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise ProtobufDataTypeFieldNotUnique 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return entry_types[0] 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetEntryTypesFromSpecifics(specifics): 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine the sync types indicated by an EntitySpecifics's field(s). 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) If the specifics have more than one recognized data type field (as commonly 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) happens with the requested_types field of GetUpdatesMessage), all types 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) will be returned. Callers must handle the possibility of the returned 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) value having more than one item. 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) specifics: A EntitySpecifics protobuf message whose extensions to 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) enumerate. 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A list of the sync types (values from ALL_TYPES) associated with each 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) recognized extension of the specifics message. 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return [data_type for data_type, field_descriptor 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) in SYNC_TYPE_TO_DESCRIPTOR.iteritems() 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if specifics.HasField(field_descriptor.name)] 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SyncTypeToProtocolDataTypeId(data_type): 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Convert from a sync type (python enum) to the protocol's data type id.""" 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SYNC_TYPE_TO_DESCRIPTOR[data_type].number 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ProtocolDataTypeIdToSyncType(protocol_data_type_id): 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Convert from the protocol's data type id to a sync type (python enum).""" 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, field_descriptor in SYNC_TYPE_TO_DESCRIPTOR.iteritems(): 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if field_descriptor.number == protocol_data_type_id: 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return data_type 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise DataTypeIdNotRecognized 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def DataTypeStringToSyncTypeLoose(data_type_string): 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Converts a human-readable string to a sync type (python enum). 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Capitalization and pluralization don't matter; this function is appropriate 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for values that might have been typed by a human being; e.g., command-line 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) flags or query parameters. 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type_string.isdigit(): 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ProtocolDataTypeIdToSyncType(int(data_type_string)) 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name = data_type_string.lower().rstrip('s') 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, field_descriptor in SYNC_TYPE_TO_DESCRIPTOR.iteritems(): 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if field_descriptor.name.lower().rstrip('s') == name: 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return data_type 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise DataTypeIdNotRecognized 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def MakeNewKeystoreKey(): 2502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Returns a new random keystore key.""" 2512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return ''.join(random.choice(string.ascii_uppercase + string.digits) 2522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for x in xrange(KEYSTORE_KEY_LENGTH)) 2532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SyncTypeToString(data_type): 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Formats a sync type enum (from ALL_TYPES) to a human-readable string.""" 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SYNC_TYPE_TO_DESCRIPTOR[data_type].name 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def CallerInfoToString(caller_info_source): 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Formats a GetUpdatesSource enum value to a readable string.""" 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return get_updates_caller_info_pb2.GetUpdatesCallerInfo \ 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) .DESCRIPTOR.enum_types_by_name['GetUpdatesSource'] \ 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) .values_by_number[caller_info_source].name 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ShortDatatypeListSummary(data_types): 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Formats compactly a list of sync types (python enums) for human eyes. 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This function is intended for use by logging. If the list of datatypes 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) contains almost all of the values, the return value will be expressed 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) in terms of the datatypes that aren't set. 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) included = set(data_types) - set([TOP_LEVEL]) 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not included: 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 'nothing' 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) excluded = set(ALL_TYPES) - included - set([TOP_LEVEL]) 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not excluded: 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 'everything' 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_text = '+'.join(sorted([SyncTypeToString(x) for x in included])) 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) all_but_text = 'all except %s' % ( 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '+'.join(sorted([SyncTypeToString(x) for x in excluded]))) 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(included) < len(excluded) or len(simple_text) <= len(all_but_text): 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return simple_text 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return all_but_text 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetDefaultEntitySpecifics(data_type): 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Get an EntitySpecifics having a sync type's default field value.""" 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) specifics = sync_pb2.EntitySpecifics() 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type in SYNC_TYPE_TO_DESCRIPTOR: 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) descriptor = SYNC_TYPE_TO_DESCRIPTOR[data_type] 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) getattr(specifics, descriptor.name).SetInParent() 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return specifics 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class PermanentItem(object): 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A specification of one server-created permanent item. 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Attributes: 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tag: A known-to-the-client value that uniquely identifies a server-created 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) permanent item. 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name: The human-readable display name for this item. 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag: The tag of the permanent item's parent. If ROOT_ID, indicates 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a top-level item. Otherwise, this must be the tag value of some other 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server-created permanent item. 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_type: A value from ALL_TYPES, giving the datatype of this permanent 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) item. This controls which types of client GetUpdates requests will 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cause the permanent item to be created and returned. 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) create_by_default: Whether the permanent item is created at startup or not. 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This value is set to True in the default case. Non-default permanent items 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) are those that are created only when a client explicitly tells the server 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to do so. 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, tag, name, parent_tag, sync_type, create_by_default=True): 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.tag = tag 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.name = name 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.parent_tag = parent_tag 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_type = sync_type 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.create_by_default = create_by_default 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class MigrationHistory(object): 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A record of the migration events associated with an account. 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Each migration event invalidates one or more datatypes on all clients 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) that had synced the datatype before the event. Such clients will continue 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to receive MigrationDone errors until they throw away their progress and 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) re-sync that datatype from the beginning. 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migrations = {} 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for datatype in ALL_TYPES: 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migrations[datatype] = [1] 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._next_migration_version = 2 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetLatestVersion(self, datatype): 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._migrations[datatype][-1] 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckAllCurrent(self, versions_map): 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises an error if any the provided versions are out of date. 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This function intentionally returns migrations in the order that they were 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) triggered. Doing it this way allows the client to queue up two migrations 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) in a row, so the second one is received while responding to the first. 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Arguments: 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) version_map: a map whose keys are datatypes and whose values are versions. 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MigrationDoneError: if a mismatch is found. 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) problems = {} 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for datatype, client_migration in versions_map.iteritems(): 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for server_migration in self._migrations[datatype]: 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if client_migration < server_migration: 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) problems.setdefault(server_migration, []).append(datatype) 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if problems: 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise MigrationDoneError(problems[min(problems.keys())]) 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def Bump(self, datatypes): 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Add a record of a migration, to cause errors on future requests.""" 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for idx, datatype in enumerate(datatypes): 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migrations[datatype].append(self._next_migration_version) 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._next_migration_version += 1 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class UpdateSieve(object): 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A filter to remove items the client has already seen.""" 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, request, migration_history=None): 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._original_request = request 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state = {} 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_history = migration_history or MigrationHistory() 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check = {} 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if request.from_progress_marker: 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for marker in request.from_progress_marker: 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) data_type = ProtocolDataTypeIdToSyncType(marker.data_type_id) 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if marker.HasField('timestamp_token_for_migration'): 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp = marker.timestamp_token_for_migration 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if timestamp: 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check[data_type] = 1 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif marker.token: 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (timestamp, version) = pickle.loads(marker.token) 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check[data_type] = version 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif marker.HasField('token'): 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp = 0 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise ValueError('No timestamp information in progress marker.') 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) data_type = ProtocolDataTypeIdToSyncType(marker.data_type_id) 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state[data_type] = timestamp 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif request.HasField('from_timestamp'): 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type in GetEntryTypesFromSpecifics(request.requested_types): 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state[data_type] = request.from_timestamp 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check[data_type] = 1 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._state: 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state[TOP_LEVEL] = min(self._state.itervalues()) 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def SummarizeRequest(self): 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamps = {} 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, timestamp in self._state.iteritems(): 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type == TOP_LEVEL: 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamps.setdefault(timestamp, []).append(data_type) 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ', '.join('<%s>@%d' % (ShortDatatypeListSummary(types), stamp) 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for stamp, types in sorted(timestamps.iteritems())) 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckMigrationState(self): 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_history.CheckAllCurrent(self._migration_versions_to_check) 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ClientWantsItem(self, item): 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return true if the client hasn't already seen an item.""" 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._state.get(GetEntryType(item), sys.maxint) < item.version 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HasAnyTimestamp(self): 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return true if at least one datatype was requested.""" 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return bool(self._state) 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetMinTimestamp(self): 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return true the smallest timestamp requested across all datatypes.""" 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return min(self._state.itervalues()) 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetFirstTimeTypes(self): 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return a list of datatypes requesting updates from timestamp zero.""" 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return [datatype for datatype, timestamp in self._state.iteritems() 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if timestamp == 0] 4285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def GetCreateMobileBookmarks(self): 4302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Return true if the client has requested to create the 'Mobile Bookmarks' 4312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) folder. 4322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 4332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return (self._original_request.HasField('create_mobile_bookmarks_folder') 4342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) and self._original_request.create_mobile_bookmarks_folder) 4352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def SaveProgress(self, new_timestamp, get_updates_response): 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Write the new_timestamp or new_progress_marker fields to a response.""" 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._original_request.from_progress_marker: 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, old_timestamp in self._state.iteritems(): 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type == TOP_LEVEL: 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_marker = sync_pb2.DataTypeProgressMarker() 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_marker.data_type_id = SyncTypeToProtocolDataTypeId(data_type) 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) final_stamp = max(old_timestamp, new_timestamp) 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) final_migration = self._migration_history.GetLatestVersion(data_type) 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_marker.token = pickle.dumps((final_stamp, final_migration)) 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if new_marker not in self._original_request.from_progress_marker: 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) get_updates_response.new_progress_marker.add().MergeFrom(new_marker) 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif self._original_request.HasField('from_timestamp'): 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._original_request.from_timestamp < new_timestamp: 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) get_updates_response.new_timestamp = new_timestamp 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class SyncDataModel(object): 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Models the account state of one sync user.""" 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _BATCH_SIZE = 100 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Specify all the permanent items that a model might need. 4595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _PERMANENT_ITEM_SPECS = [ 4602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_apps', name='Apps', 4612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=APPS), 4622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_app_notifications', name='App Notifications', 4632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=APP_NOTIFICATION), 4642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_app_settings', 4652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='App Settings', 4662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=APP_SETTINGS), 4675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_bookmarks', name='Bookmarks', 4682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=BOOKMARK), 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('bookmark_bar', name='Bookmark Bar', 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag='google_chrome_bookmarks', sync_type=BOOKMARK), 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('other_bookmarks', name='Other Bookmarks', 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag='google_chrome_bookmarks', sync_type=BOOKMARK), 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('synced_bookmarks', name='Synced Bookmarks', 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag='google_chrome_bookmarks', sync_type=BOOKMARK, 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) create_by_default=False), 4765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_autofill', name='Autofill', 4772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=AUTOFILL), 4785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_autofill_profiles', name='Autofill Profiles', 4792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=AUTOFILL_PROFILE), 4802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_device_info', name='Device Info', 4812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=DEVICE_INFO), 4822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_experiments', name='Experiments', 4832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=EXPERIMENTS), 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_extension_settings', 4855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name='Extension Settings', 4862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=EXTENSION_SETTINGS), 4875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_extensions', name='Extensions', 4882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=EXTENSIONS), 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_history_delete_directives', 4905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name='History Delete Directives', 4912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, 4925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_type=HISTORY_DELETE_DIRECTIVE), 4932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_favicon_images', 4942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='Favicon Images', 4952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, 4962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sync_type=FAVICON_IMAGES), 4972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_favicon_tracking', 4982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='Favicon Tracking', 4992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, 5002a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sync_type=FAVICON_TRACKING), 501c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PermanentItem('google_chrome_managed_user_settings', 502c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) name='Managed User Settings', 503c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=MANAGED_USER_SETTING), 50490dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) PermanentItem('google_chrome_managed_users', 50590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) name='Managed Users', 50690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=MANAGED_USER), 5072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_nigori', name='Nigori', 5082a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=NIGORI), 5095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_passwords', name='Passwords', 5102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=PASSWORD), 5112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_preferences', name='Preferences', 5122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=PREFERENCE), 513c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PermanentItem('google_chrome_priority_preferences', 514c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) name='Priority Preferences', 515c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=PRIORITY_PREFERENCE), 5162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_synced_notifications', 5172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='Synced Notifications', 5182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=SYNCED_NOTIFICATION), 5195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_search_engines', name='Search Engines', 5202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=SEARCH_ENGINE), 5215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_sessions', name='Sessions', 5222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=SESSION), 5235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_themes', name='Themes', 5242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=THEME), 5255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_typed_urls', name='Typed URLs', 5262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=TYPED_URL), 5272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_dictionary', name='Dictionary', 5282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=DICTIONARY), 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ] 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Monotonically increasing version number. The next object change will 5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # take on this value + 1. 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._version = 0 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The definitive copy of this client's items: a map from ID string to a 5375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # SyncEntity protocol buffer. 5385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries = {} 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.ResetStoreBirthday() 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.migration_history = MigrationHistory() 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error = sync_pb2.ClientToServerResponse.Error() 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error_frequency = 0 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count_before_errors = 0 545868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) self.acknowledge_managed_users = False 5462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._keys = [MakeNewKeystoreKey()] 5475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _SaveEntry(self, entry): 5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Insert or update an entry in the change log, and give it a new version. 5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The ID fields of this entry are assumed to be valid server IDs. This 5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry will be updated with a new version number and sync_timestamp. 5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: The entry to be added or updated. 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._version += 1 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Maintain a global (rather than per-item) sequence number and use it 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # both as the per-entry version as well as the update-progress timestamp. 5605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This simulates the behavior of the original server implementation. 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.version = self._version 5625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.sync_timestamp = self._version 5635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Preserve the originator info, which the client is not required to send 5655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # when updating. 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base_entry = self._entries.get(entry.id_string) 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if base_entry: 5685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_cache_guid = base_entry.originator_cache_guid 5695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_client_item_id = base_entry.originator_client_item_id 5705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string] = copy.deepcopy(entry) 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ServerTagToId(self, tag): 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine the server ID from a server-unique tag. 5755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting value is guaranteed not to collide with the other ID 5775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generation methods. 5785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype: The sync type (python enum) of the identified object. 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tag: The unique, known-to-the-client tag of a server-generated item. 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The string value of the computed server ID. 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 5855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not tag or tag == ROOT_ID: 5865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tag 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) spec = [x for x in self._PERMANENT_ITEM_SPECS if x.tag == tag][0] 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(spec.sync_type, '<server tag>%s' % tag) 5895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ClientTagToId(self, datatype, tag): 5915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine the server ID from a client-unique tag. 5925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting value is guaranteed not to collide with the other ID 5945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generation methods. 5955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype: The sync type (python enum) of the identified object. 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tag: The unique, opaque-to-the-server tag of a client-tagged item. 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The string value of the computed server ID. 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(datatype, '<client tag>%s' % tag) 6035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ClientIdToId(self, datatype, client_guid, client_item_id): 6055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Compute a unique server ID from a client-local ID tag. 6065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting value is guaranteed not to collide with the other ID 6085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generation methods. 6095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype: The sync type (python enum) of the identified object. 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_guid: A globally unique ID that identifies the client which 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) created this item. 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_item_id: An ID that uniquely identifies this item on the client 6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) which created it. 6165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 6175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The string value of the computed server ID. 6185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Using the client ID info is not required here (we could instead generate 6205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # a random ID), but it's useful for debugging. 6215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(datatype, 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<server ID originally>%s/%s' % (client_guid, client_item_id)) 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _MakeCurrentId(self, datatype, inner_id): 6255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return '%d^%d^%s' % (datatype, 6265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.migration_history.GetLatestVersion(datatype), 6275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) inner_id) 6285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ExtractIdInfo(self, id_string): 6305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not id_string or id_string == ROOT_ID: 6315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 6325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype_string, separator, remainder = id_string.partition('^') 6335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) migration_version_string, separator, inner_id = remainder.partition('^') 6345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (int(datatype_string), int(migration_version_string), inner_id) 6355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _WritePosition(self, entry, parent_id): 6375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Ensure the entry has an absolute, numeric position and parent_id. 6385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Historically, clients would specify positions using the predecessor-based 6405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) references in the insert_after_item_id field; starting July 2011, this 6415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) was changed and Chrome now sends up the absolute position. The server 6425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) must store a position_in_parent value and must not maintain 6435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) insert_after_item_id. 644c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) Starting in Jan 2013, the client will also send up a unique_position field 645c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) which should be saved and returned on subsequent GetUpdates. 6465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: The entry for which to write a position. Its ID field are 649c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) assumed to be server IDs. This entry will have its parent_id_string, 650c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) position_in_parent and unique_position fields updated; its 651c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) insert_after_item_id field will be cleared. 6525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_id: The ID of the entry intended as the new parent. 6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string = parent_id 6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not entry.HasField('position_in_parent'): 6575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.position_in_parent = 1337 # A debuggable, distinctive default. 6585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.ClearField('insert_after_item_id') 6595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ItemExists(self, id_string): 6615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine whether an item exists in the changelog.""" 6625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return id_string in self._entries 6635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CreatePermanentItem(self, spec): 6655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Create one permanent item from its spec, if it doesn't exist. 6665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting item is added to the changelog. 6685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) spec: A PermanentItem object holding the properties of the item to create. 6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) id_string = self._ServerTagToId(spec.tag) 6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._ItemExists(id_string): 6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Creating permanent item: %s' % spec.name 6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry = sync_pb2.SyncEntity() 6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.id_string = id_string 6785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.non_unique_name = spec.name 6795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.name = spec.name 6805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.server_defined_unique_tag = spec.tag 6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.folder = True 6825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.deleted = False 6835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.specifics.CopyFrom(GetDefaultEntitySpecifics(spec.sync_type)) 6845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._WritePosition(entry, self._ServerTagToId(spec.parent_tag)) 6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(entry) 6865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CreateDefaultPermanentItems(self, requested_types): 6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Ensure creation of all default permanent items for a given set of types. 6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) requested_types: A list of sync data types from ALL_TYPES. 6925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) All default permanent items of only these types will be created. 6935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for spec in self._PERMANENT_ITEM_SPECS: 6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if spec.sync_type in requested_types and spec.create_by_default: 6965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._CreatePermanentItem(spec) 6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ResetStoreBirthday(self): 6995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Resets the store birthday to a random value.""" 7005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): uuid.uuid1() is better, but python 2.5 only. 7015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.store_birthday = '%0.30f' % random.random() 7025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def StoreBirthday(self): 7045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Gets the store birthday.""" 7055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.store_birthday 7065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetChanges(self, sieve): 7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Get entries which have changed, oldest first. 7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The returned entries are limited to being _BATCH_SIZE many. The entries 7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) are returned in strict version order. 7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sieve: An update sieve to use to filter out updates the client 7155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) has already seen. 7165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 7175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A tuple of (version, entries, changes_remaining). Version is a new 7185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp value, which should be used as the starting point for the 7195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) next query. Entries is the batch of entries meeting the current 7205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp query. Changes_remaining indicates the number of changes 7215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) left on the server after this batch. 7225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not sieve.HasAnyTimestamp(): 7245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (0, [], 0) 7255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) min_timestamp = sieve.GetMinTimestamp() 7262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) first_time_types = sieve.GetFirstTimeTypes() 7272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._CreateDefaultPermanentItems(first_time_types) 7282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Mobile bookmark folder is not created by default, create it only when 7292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # client requested it. 7302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (sieve.GetCreateMobileBookmarks() and 7312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) first_time_types.count(BOOKMARK) > 0): 7322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.TriggerCreateSyncedBookmarks() 7332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 734868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) self.TriggerAcknowledgeManagedUsers() 735868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 7365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) change_log = sorted(self._entries.values(), 7375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) key=operator.attrgetter('version')) 7385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_changes = [x for x in change_log if x.version > min_timestamp] 7395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Pick batch_size new changes, and then filter them. This matches 7405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # the RPC behavior of the production sync server. 7415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) batch = new_changes[:self._BATCH_SIZE] 7425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not batch: 7435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Client is up to date. 7445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (min_timestamp, [], 0) 7455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Restrict batch to requested types. Tombstones are untyped 7475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # and will always get included. 7485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filtered = [copy.deepcopy(item) for item in batch 7495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if item.deleted or sieve.ClientWantsItem(item)] 7505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The new client timestamp is the timestamp of the last item in the 7525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # batch, even if that item was filtered out. 7535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (batch[-1].version, filtered, len(new_changes) - len(batch)) 7545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def GetKeystoreKeys(self): 7562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Returns the encryption keys for this account.""" 7572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) print "Returning encryption keys: %s" % self._keys 7582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return self._keys 7595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CopyOverImmutableFields(self, entry): 7615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Preserve immutable fields by copying pre-commit state. 7625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A sync entity from the client. 7655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.id_string in self._entries: 7675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._entries[entry.id_string].HasField( 7685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'server_defined_unique_tag'): 7695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.server_defined_unique_tag = ( 7705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string].server_defined_unique_tag) 7715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CheckVersionForCommit(self, entry): 7735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Perform an optimistic concurrency check on the version number. 7745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Clients are only allowed to commit if they report having seen the most 7765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) recent version of an object. 7775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A sync entity from the client. It is assumed that ID fields 7805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) have been converted to server IDs. 7815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 7825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A boolean value indicating whether the client's version matches the 7835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) newest server version for the given entry. 7845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.id_string in self._entries: 7865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Allow edits/deletes if the version matches, and any undeletion. 7875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (self._entries[entry.id_string].version == entry.version or 7885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string].deleted) 7895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 7905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Allow unknown ID only if the client thinks it's new too. 7915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return entry.version == 0 7925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CheckParentIdForCommit(self, entry): 7945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Check that the parent ID referenced in a SyncEntity actually exists. 7955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A sync entity from the client. It is assumed that ID fields 7985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) have been converted to server IDs. 7995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 8005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A boolean value indicating whether the entity's parent ID is an object 8015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) that actually exists (and is not deleted) in the current account state. 8025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string == ROOT_ID: 8045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This is generally allowed. 8055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 8065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string not in self._entries: 8075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Warning: Client sent unknown ID. Should never happen.' 8085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string == entry.id_string: 8105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Warning: Client sent circular reference. Should never happen.' 8115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._entries[entry.parent_id_string].deleted: 8135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This can happen in a race condition between two clients. 8145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self._entries[entry.parent_id_string].folder: 8165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Warning: Client sent non-folder parent. Should never happen.' 8175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 8195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _RewriteIdsAsServerIds(self, entry, cache_guid, commit_session): 8215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Convert ID fields in a client sync entry to server IDs. 8225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A commit batch sent by a client may contain new items for which the 8245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server has not generated IDs yet. And within a commit batch, later 8255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) items are allowed to refer to earlier items. This method will 8265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generate server IDs for new items, as well as rewrite references 8275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to items whose server IDs were generated earlier in the batch. 8285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 8305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: The client sync entry to modify. 8315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cache_guid: The globally unique ID of the client that sent this 8325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit request. 8335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_session: A dictionary mapping the original IDs to the new server 8345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) IDs, for any items committed earlier in the batch. 8355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.version == 0: 8375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) data_type = GetEntryType(entry) 8385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.HasField('client_defined_unique_tag'): 8395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # When present, this should determine the item's ID. 8405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_id = self._ClientTagToId(data_type, entry.client_defined_unique_tag) 8415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_id = self._ClientIdToId(data_type, cache_guid, entry.id_string) 8435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_cache_guid = cache_guid 8445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_client_item_id = entry.id_string 8455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_session[entry.id_string] = new_id # Remember the remapping. 8465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.id_string = new_id 8475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string in commit_session: 8485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string = commit_session[entry.parent_id_string] 8495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.insert_after_item_id in commit_session: 8505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.insert_after_item_id = commit_session[entry.insert_after_item_id] 8515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ValidateCommitEntries(self, entries): 8535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raise an exception if a commit batch contains any global errors. 8545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Arguments: 8565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entries: an iterable containing commit-form SyncEntity protocol buffers. 8575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 8595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MigrationDoneError: if any of the entries reference a recently-migrated 8605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype. 8615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_ids_in_commit = set() 8635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) local_ids_in_commit = set() 8645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in entries: 8655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.version: 8665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_ids_in_commit.add(entry.id_string) 8675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) local_ids_in_commit.add(entry.id_string) 8695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.HasField('parent_id_string'): 8705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string not in local_ids_in_commit: 8715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_ids_in_commit.add(entry.parent_id_string) 8725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) versions_present = {} 8745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for server_id in server_ids_in_commit: 8755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed = self._ExtractIdInfo(server_id) 8765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if parsed: 8775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype, version, _ = parsed 8785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) versions_present.setdefault(datatype, []).append(version) 8795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.migration_history.CheckAllCurrent( 8815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) dict((k, min(v)) for k, v in versions_present.iteritems())) 8825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CommitEntry(self, entry, cache_guid, commit_session): 8845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Attempt to commit one entry to the user's account. 8855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 8875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A SyncEntity protobuf representing desired object changes. 8885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cache_guid: A string value uniquely identifying the client; this 8895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) is used for ID generation and will determine the originator_cache_guid 8905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if the entry is new. 8915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_session: A dictionary mapping client IDs to server IDs for any 8925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) objects committed earlier this session. If the entry gets a new ID 8935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) during commit, the change will be recorded here. 8945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 8955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A SyncEntity reflecting the post-commit value of the entry, or None 8965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if the entry was not committed due to an error. 8975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry = copy.deepcopy(entry) 8995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Generate server IDs for this entry, and write generated server IDs 9015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # from earlier entries into the message's fields, as appropriate. The 9025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # ID generation state is stored in 'commit_session'. 9035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._RewriteIdsAsServerIds(entry, cache_guid, commit_session) 9045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Perform the optimistic concurrency check on the entry's version number. 9065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Clients are not allowed to commit unless they indicate that they've seen 9075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # the most recent version of an object. 9085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self._CheckVersionForCommit(entry): 9095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 9105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Check the validity of the parent ID; it must exist at this point. 9125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): Implement cycle detection and resolution. 9135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self._CheckParentIdForCommit(entry): 9145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 9155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._CopyOverImmutableFields(entry); 9175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # At this point, the commit is definitely going to happen. 9195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Deletion works by storing a limited record for an entry, called a 9215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # tombstone. A sync server must track deleted IDs forever, since it does 9225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # not keep track of client knowledge (there's no deletion ACK event). 9235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.deleted: 9245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def MakeTombstone(id_string): 9255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Make a tombstone entry that will replace the entry being deleted. 9265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 9285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) id_string: Index of the SyncEntity to be deleted. 9295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 9305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A new SyncEntity reflecting the fact that the entry is deleted. 9315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 9325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Only the ID, version and deletion state are preserved on a tombstone. 9335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): Does the production server not preserve the type? Not 9345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # doing so means that tombstones cannot be filtered based on 9355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # requested_types at GetUpdates time. 9365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone = sync_pb2.SyncEntity() 9375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone.id_string = id_string 9385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone.deleted = True 9395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone.name = '' 9405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tombstone 9415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def IsChild(child_id): 9435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Check if a SyncEntity is a child of entry, or any of its children. 9445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 9465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) child_id: Index of the SyncEntity that is a possible child of entry. 9475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 9485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) True if it is a child; false otherwise. 9495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 9505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if child_id not in self._entries: 9515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 9525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._entries[child_id].parent_id_string == entry.id_string: 9535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 9545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return IsChild(self._entries[child_id].parent_id_string) 9555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Identify any children entry might have. 9575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) child_ids = [child.id_string for child in self._entries.itervalues() 9585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if IsChild(child.id_string)] 9595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Mark all children that were identified as deleted. 9615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for child_id in child_ids: 9625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(MakeTombstone(child_id)) 9635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Delete entry itself. 9655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry = MakeTombstone(entry.id_string) 9665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 9675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Comments in sync.proto detail how the representation of positional 968c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # ordering works. 969c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 970c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # We've almost fully deprecated the 'insert_after_item_id' field. 971c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # The 'position_in_parent' field is also deprecated, but as of Jan 2013 972c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # is still in common use. The 'unique_position' field is the latest 973c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # and greatest in positioning technology. 974c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 975c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # This server supports 'position_in_parent' and 'unique_position'. 9765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._WritePosition(entry, entry.parent_id_string) 9775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Preserve the originator info, which the client is not required to send 9795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # when updating. 9805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base_entry = self._entries.get(entry.id_string) 9815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if base_entry and not entry.HasField('originator_cache_guid'): 9825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_cache_guid = base_entry.originator_cache_guid 9835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_client_item_id = base_entry.originator_client_item_id 9845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Store the current time since the Unix epoch in milliseconds. 9865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.mtime = (int((time.mktime(time.gmtime()) - 9875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) time.mktime(UNIX_TIME_EPOCH))*1000)) 9885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Commit the change. This also updates the version number. 9905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(entry) 9915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return entry 9925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _RewriteVersionInId(self, id_string): 9945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Rewrites an ID so that its migration version becomes current.""" 9955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed_id = self._ExtractIdInfo(id_string) 9965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not parsed_id: 9975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return id_string 9985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype, old_migration_version, inner_id = parsed_id 9995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(datatype, inner_id) 10005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def TriggerMigration(self, datatypes): 10025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Cause a migration to occur for a set of datatypes on this account. 10035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Clients will see the MIGRATION_DONE error for these datatypes until they 10055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) resync them. 10065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 10075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) versions_to_remap = self.migration_history.Bump(datatypes) 10085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) all_entries = self._entries.values() 10095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries.clear() 10105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in all_entries: 10115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_id = self._RewriteVersionInId(entry.id_string) 10125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.id_string = new_id 10135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.HasField('parent_id_string'): 10145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string = self._RewriteVersionInId( 10155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string) 10165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string] = entry 10175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def TriggerSyncTabFavicons(self): 10195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Set the 'sync_tab_favicons' field to this account's nigori node. 10205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) If the field is not currently set, will write a new nigori node entry 10225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) with the field set. Else does nothing. 10235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 10245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_tag = "google_chrome_nigori" 10265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_original = self._entries.get(self._ServerTagToId(nigori_tag)) 10275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (nigori_original.specifics.nigori.sync_tab_favicons): 10285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 10295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_new = copy.deepcopy(nigori_original) 10305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_new.specifics.nigori.sync_tabs = True 10315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(nigori_new) 10325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def TriggerCreateSyncedBookmarks(self): 10345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Create the Synced Bookmarks folder under the Bookmarks permanent item. 10355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Clients will then receive the Synced Bookmarks folder on future 10375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) GetUpdates, and new bookmarks can be added within the Synced Bookmarks 10385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) folder. 10395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 10405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) synced_bookmarks_spec, = [spec for spec in self._PERMANENT_ITEM_SPECS 10425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if spec.name == "Synced Bookmarks"] 10435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._CreatePermanentItem(synced_bookmarks_spec) 10445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def TriggerEnableKeystoreEncryption(self): 10462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Create the keystore_encryption experiment entity and enable it. 10472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) A new entity within the EXPERIMENTS datatype is created with the unique 10492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) client tag "keystore_encryption" if it doesn't already exist. The 10502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_encryption message is then filled with |enabled| set to true. 10512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 10522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) experiment_id = self._ServerTagToId("google_chrome_experiments") 10542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_encryption_id = self._ClientTagToId( 10552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPERIMENTS, 10562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) KEYSTORE_ENCRYPTION_EXPERIMENT_TAG) 10572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry = self._entries.get(keystore_encryption_id) 10582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if keystore_entry is None: 10592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry = sync_pb2.SyncEntity() 10602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.id_string = keystore_encryption_id 10612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.name = "Keystore Encryption" 10622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.client_defined_unique_tag = ( 10632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) KEYSTORE_ENCRYPTION_EXPERIMENT_TAG) 10642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.folder = False 10652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.deleted = False 10662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.specifics.CopyFrom(GetDefaultEntitySpecifics(EXPERIMENTS)) 10672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._WritePosition(keystore_entry, experiment_id) 10682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.specifics.experiments.keystore_encryption.enabled = True 10702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._SaveEntry(keystore_entry) 10722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def TriggerRotateKeystoreKeys(self): 10742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Rotate the current set of keystore encryption keys. 10752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) |self._keys| will have a new random encryption key appended to it. We touch 10772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) the nigori node so that each client will receive the new encryption keys 10782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) only once. 10792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 10802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Add a new encryption key. 10822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._keys += [MakeNewKeystoreKey(), ] 10832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Increment the nigori node's timestamp, so clients will get the new keys 10852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # on their next GetUpdates (any time the nigori node is sent back, we also 10862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # send back the keystore keys). 10872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) nigori_tag = "google_chrome_nigori" 10882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._SaveEntry(self._entries.get(self._ServerTagToId(nigori_tag))) 10892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1090868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def TriggerAcknowledgeManagedUsers(self): 1091868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) """Set the "acknowledged" flag for any managed user entities that don't have 1092868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) it set already. 1093868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) """ 1094868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 1095868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if not self.acknowledge_managed_users: 1096868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return 1097868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 1098868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) managed_users = [copy.deepcopy(entry) for entry in self._entries.values() 1099868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) if entry.specifics.HasField('managed_user') 1100868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) and not entry.specifics.managed_user.acknowledged] 1101868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) for user in managed_users: 1102868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) user.specifics.managed_user.acknowledged = True 1103868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) self._SaveEntry(user) 1104868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 1105ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch def TriggerEnablePreCommitGetUpdateAvoidance(self): 1106ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """Sets the experiment to enable pre-commit GetUpdate avoidance.""" 1107ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch experiment_id = self._ServerTagToId("google_chrome_experiments") 1108ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch pre_commit_gu_avoidance_id = self._ClientTagToId( 1109ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch EXPERIMENTS, 1110ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch PRE_COMMIT_GU_AVOIDANCE_EXPERIMENT_TAG) 1111ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry = self._entries.get(pre_commit_gu_avoidance_id) 1112ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch if entry is None: 1113ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry = sync_pb2.SyncEntity() 1114ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.id_string = pre_commit_gu_avoidance_id 1115ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.name = "Pre-commit GU avoidance" 1116ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.client_defined_unique_tag = PRE_COMMIT_GU_AVOIDANCE_EXPERIMENT_TAG 1117ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.folder = False 1118ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.deleted = False 1119ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.specifics.CopyFrom(GetDefaultEntitySpecifics(EXPERIMENTS)) 1120ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch self._WritePosition(entry, experiment_id) 1121ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch entry.specifics.experiments.pre_commit_update_avoidance.enabled = True 1122ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch self._SaveEntry(entry) 1123ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 11245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def SetInducedError(self, error, error_frequency, 11255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_count_before_errors): 11265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error = error 11275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error_frequency = error_frequency 11285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count_before_errors = sync_count_before_errors 11295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetInducedError(self): 11315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.induced_error 11325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class TestServer(object): 11355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """An object to handle requests for one (and only one) Chrome Sync account. 11365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TestServer consumes the sync command messages that are the outermost 11385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) layers of the protocol, performs the corresponding actions on its 11392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SyncDataModel, and constructs an appropriate response message. 11405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 11415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 11435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The implementation supports exactly one account; its state is here. 11445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account = SyncDataModel() 11455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock = threading.Lock() 11465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Clients that have talked to us: a map from the full client ID 11475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # to its nickname. 11485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.clients = {} 11495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.client_name_generator = ('+' * times + chr(c) 11505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for times in xrange(0, sys.maxint) for c in xrange(ord('A'), ord('Z'))) 11515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.transient_error = False 11525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count = 0 11535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetShortClientName(self, query): 11555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed = cgi.parse_qs(query[query.find('?')+1:]) 11565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_id = parsed.get('client_id') 11575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not client_id: 11585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return '?' 11595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_id = client_id[0] 11605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if client_id not in self.clients: 11615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.clients[client_id] = self.client_name_generator.next() 11625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.clients[client_id] 11635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckStoreBirthday(self, request): 11655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises StoreBirthdayError if the request's birthday is a mismatch.""" 11665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not request.HasField('store_birthday'): 11675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 11685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.account.StoreBirthday() != request.store_birthday: 11695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise StoreBirthdayError 11705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckTransientError(self): 11725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises TransientError if transient_error variable is set.""" 11735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.transient_error: 11745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise TransientError 11755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckSendError(self): 11775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises SyncInducedError if needed.""" 11785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (self.account.induced_error.error_type != 11795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_enums_pb2.SyncEnums.UNKNOWN): 11805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Always means return the given error for all requests. 11815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.account.induced_error_frequency == ERROR_FREQUENCY_ALWAYS: 11825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise SyncInducedError 11835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This means the FIRST 2 requests of every 3 requests 11845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # return an error. Don't switch the order of failures. There are 11855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # test cases that rely on the first 2 being the failure rather than 11865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # the last 2. 11875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif (self.account.induced_error_frequency == 11885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_TWO_THIRDS): 11895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (((self.sync_count - 11905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.sync_count_before_errors) % 3) != 0): 11915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise SyncInducedError 11925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise InducedErrorFrequencyNotDefined 11945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleMigrate(self, path): 11965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) query = urlparse.urlparse(path)[4] 11975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 200 11985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.acquire() 11995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatypes = [DataTypeStringToSyncTypeLoose(x) 12015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for x in urlparse.parse_qs(query).get('type',[])] 12025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if datatypes: 12035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.TriggerMigration(datatypes) 12045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Migrated datatypes %s' % ( 12055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ' and '.join(SyncTypeToString(x).upper() for x in datatypes)) 12065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 12075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Please specify one or more <i>type=name</i> parameters' 12085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 400 12095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except DataTypeIdNotRecognized, error: 12105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Could not interpret datatype name' 12115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 400 12125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 12135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.release() 12145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (code, '<html><title>Migration: %d</title><H1>%d %s</H1></html>' % 12155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (code, code, response)) 12165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleSetInducedError(self, path): 12185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) query = urlparse.urlparse(path)[4] 12195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.acquire() 12205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 200 12215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Success' 12225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error = sync_pb2.ClientToServerResponse.Error() 12235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error_type = urlparse.parse_qs(query)['error'] 12255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) action = urlparse.parse_qs(query)['action'] 12265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_type = int(error_type[0]) 12275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.action = int(action[0]) 12285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.url = (urlparse.parse_qs(query)['url'])[0] 12305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 12315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.url = '' 12325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_description =( 12345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (urlparse.parse_qs(query)['error_description'])[0]) 12355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 12365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_description = '' 12375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error_frequency = int((urlparse.parse_qs(query)['frequency'])[0]) 12395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 12405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error_frequency = ERROR_FREQUENCY_ALWAYS 12415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.SetInducedError(error, error_frequency, self.sync_count) 12425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = ('Error = %d, action = %d, url = %s, description = %s' % 12435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (error.error_type, error.action, 12445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.url, 12455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_description)) 12465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except error: 12475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Could not parse url' 12485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 400 12495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 12505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.release() 12515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (code, '<html><title>SetError: %d</title><H1>%d %s</H1></html>' % 12525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (code, code, response)) 12535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCreateBirthdayError(self): 12555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.ResetStoreBirthday() 12565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Birthday error</title><H1>Birthday error</H1></html>') 12595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleSetTransientError(self): 12615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.transient_error = True 12625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Transient error</title><H1>Transient error</H1></html>') 12655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleSetSyncTabFavicons(self): 12675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Set 'sync_tab_favicons' field of the nigori node for this account.""" 12685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.TriggerSyncTabFavicons() 12695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Tab Favicons</title><H1>Tab Favicons</H1></html>') 12725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCreateSyncedBookmarks(self): 12745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Create the Synced Bookmarks folder under Bookmarks.""" 12755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.TriggerCreateSyncedBookmarks() 12765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Synced Bookmarks</title><H1>Synced Bookmarks</H1></html>') 12795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def HandleEnableKeystoreEncryption(self): 12812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Enables the keystore encryption experiment.""" 12822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.account.TriggerEnableKeystoreEncryption() 12832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return ( 12842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 200, 12852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<html><title>Enable Keystore Encryption</title>' 12862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<H1>Enable Keystore Encryption</H1></html>') 12872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 12882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def HandleRotateKeystoreKeys(self): 12892a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Rotate the keystore encryption keys.""" 12902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.account.TriggerRotateKeystoreKeys() 12912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return ( 12922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 200, 12932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<html><title>Rotate Keystore Keys</title>' 12942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<H1>Rotate Keystore Keys</H1></html>') 12952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1296868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) def HandleEnableManagedUserAcknowledgement(self): 1297868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) """Enable acknowledging newly created managed users.""" 1298868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) self.account.acknowledge_managed_users = True 1299868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return ( 1300868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 200, 1301868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) '<html><title>Enable Managed User Acknowledgement</title>' 1302868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) '<h1>Enable Managed User Acknowledgement</h1></html>') 1303868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 1304ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch def HandleEnablePreCommitGetUpdateAvoidance(self): 1305ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch """Enables the pre-commit GU avoidance experiment.""" 1306ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch self.account.TriggerEnablePreCommitGetUpdateAvoidance() 1307ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch return ( 1308ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 200, 1309ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch '<html><title>Enable pre-commit GU avoidance</title>' 1310ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch '<H1>Enable pre-commit GU avoidance</H1></html>') 1311ca12bfac764ba476d6cd062bf1dde12cc64c3f40Ben Murdoch 13125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCommand(self, query, raw_request): 13135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Decode and handle a sync command from a raw input of bytes. 13145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This is the main entry point for this class. It is safe to call this 13165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) method from multiple threads. 13175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 13195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raw_request: An iterable byte sequence to be interpreted as a sync 13205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) protocol command. 13215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 13225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A tuple (response_code, raw_response); the first value is an HTTP 13235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result code, while the second value is a string of bytes which is the 13245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) serialized reply to the command. 13255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 13265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.acquire() 13275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count += 1 13285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def print_context(direction): 13295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '[Client %s %s %s.py]' % (self.GetShortClientName(query), direction, 13305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __name__), 13315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 13335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request = sync_pb2.ClientToServerMessage() 13345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request.MergeFromString(raw_request) 13355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) contents = request.message_contents 13365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = sync_pb2.ClientToServerResponse() 13385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.SUCCESS 13395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.CheckStoreBirthday(request) 13405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.CheckTransientError() 13425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.CheckSendError() 13435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('->') 13455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if contents == sync_pb2.ClientToServerMessage.AUTHENTICATE: 13475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Authenticate' 13485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # We accept any authentication token, and support only one account. 13495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): Mock out the GAIA authentication as well; hook up here. 13505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.authenticate.user.email = 'syncjuser@chromium' 13515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.authenticate.user.display_name = 'Sync J User' 13525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif contents == sync_pb2.ClientToServerMessage.COMMIT: 13535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Commit %d item(s)' % len(request.commit.entries) 13545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.HandleCommit(request.commit, response.commit) 13555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif contents == sync_pb2.ClientToServerMessage.GET_UPDATES: 13565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'GetUpdates', 13575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.HandleGetUpdates(request.get_updates, response.get_updates) 13585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '%d update(s)' % len(response.get_updates.entries) 13605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 13615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Unrecognizable sync request!' 13625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (400, None) # Bad request. 13635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except MigrationDoneError, error: 13655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'MIGRATION_DONE: <%s>' % (ShortDatatypeListSummary(error.datatypes)) 13675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = sync_pb2.ClientToServerResponse() 13685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.MIGRATION_DONE 13705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.migrated_data_type_id[:] = [ 13715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SyncTypeToProtocolDataTypeId(x) for x in error.datatypes] 13725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except StoreBirthdayError, error: 13745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'NOT_MY_BIRTHDAY' 13765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = sync_pb2.ClientToServerResponse() 13775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.NOT_MY_BIRTHDAY 13795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except TransientError, error: 13815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ### This is deprecated now. Would be removed once test cases are removed. 13825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'TRANSIENT_ERROR' 13845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.TRANSIENT_ERROR 13865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except SyncInducedError, error: 13885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'INDUCED_ERROR' 13905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error = self.account.GetInducedError() 13925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.error_type = error.error_type 13935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.url = error.url 13945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.error_description = error.error_description 13955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.action = error.action 13965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 13985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.release() 13995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCommit(self, commit_message, commit_response): 14015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Respond to a Commit request by updating the user's account state. 14025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Commit attempts stop after the first error, returning a CONFLICT result 14045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for any unattempted entries. 14055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 14075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_message: A sync_pb.CommitMessage protobuf holding the content 14085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) of the client's request. 14095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_response: A sync_pb.CommitResponse protobuf into which a reply 14105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to the client request will be written. 14115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 14125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_response.SetInParent() 14135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) batch_failure = False 14145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) session = {} # Tracks ID renaming during the commit operation. 14155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) guid = commit_message.cache_guid 14165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.ValidateCommitEntries(commit_message.entries) 14185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in commit_message.entries: 14205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_entry = None 14215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not batch_failure: 14225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Try to commit the change to the account. 14235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_entry = self.account.CommitEntry(entry, guid, session) 14245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # An entryresponse is returned in both success and failure cases. 14265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply = commit_response.entryresponse.add() 14275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not server_entry: 14285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.response_type = sync_pb2.CommitResponse.CONFLICT 14295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.error_message = 'Conflict.' 14305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) batch_failure = True # One failure halts the batch. 14315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 14325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.response_type = sync_pb2.CommitResponse.SUCCESS 14335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # These are the properties that the server is allowed to override 14345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # during commit; the client wants to know their values at the end 14355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # of the operation. 14365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.id_string = server_entry.id_string 14375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not server_entry.deleted: 14385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Note: the production server doesn't actually send the 14395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # parent_id_string on commit responses, so we don't either. 14405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.position_in_parent = server_entry.position_in_parent 14415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.version = server_entry.version 14425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.name = server_entry.name 14435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.non_unique_name = server_entry.non_unique_name 14445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 14455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.version = entry.version + 1 14465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleGetUpdates(self, update_request, update_response): 14485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Respond to a GetUpdates request by querying the user's account. 14495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 14515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_request: A sync_pb.GetUpdatesMessage protobuf holding the content 14525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) of the client's request. 14535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_response: A sync_pb.GetUpdatesResponse protobuf into which a reply 14545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to the client request will be written. 14555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 14565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_response.SetInParent() 14575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_sieve = UpdateSieve(update_request, self.account.migration_history) 14585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print CallerInfoToString(update_request.caller_info.source), 14605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print update_sieve.SummarizeRequest() 14615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_sieve.CheckMigrationState() 14635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_timestamp, entries, remaining = self.account.GetChanges(update_sieve) 14655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_response.changes_remaining = remaining 14672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sending_nigori_node = False 14685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in entries: 14692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if entry.name == 'Nigori': 14702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sending_nigori_node = True 14715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply = update_response.entries.add() 14725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.CopyFrom(entry) 14735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_sieve.SaveProgress(new_timestamp, update_response) 14745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if update_request.need_encryption_key or sending_nigori_node: 14762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) update_response.encryption_keys.extend(self.account.GetKeystoreKeys()) 1477