chromiumsync.py revision 90dce4d38c5ff5333bea97d859d4e484e27edf0c
12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright 2013 The Chromium Authors. All rights reserved. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# found in the LICENSE file. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)"""An implementation of the server side of the Chromium sync protocol. 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)The details of the protocol are described mostly by comments in the protocol 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)buffer definition at chrome/browser/sync/protocol/sync.proto. 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)""" 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import cgi 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import copy 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import operator 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import pickle 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import random 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import string 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sys 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import threading 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import time 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import urlparse 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import app_notification_specifics_pb2 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import app_setting_specifics_pb2 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import app_specifics_pb2 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import autofill_specifics_pb2 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import bookmark_specifics_pb2 272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import dictionary_specifics_pb2 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import get_updates_caller_info_pb2 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import extension_setting_specifics_pb2 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import extension_specifics_pb2 312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import favicon_image_specifics_pb2 322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import favicon_tracking_specifics_pb2 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import history_delete_directive_specifics_pb2 34c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import managed_user_setting_specifics_pb2 3590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles)import managed_user_specifics_pb2 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import nigori_specifics_pb2 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import password_specifics_pb2 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import preference_specifics_pb2 39c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)import priority_preference_specifics_pb2 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import search_engine_specifics_pb2 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import session_specifics_pb2 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sync_pb2 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import sync_enums_pb2 442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import synced_notification_specifics_pb2 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import theme_specifics_pb2 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import typed_url_specifics_pb2 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# An enumeration of the various kinds of data that can be synced. 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Over the wire, this enumeration is not used: a sync object's type is 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# inferred by which EntitySpecifics field it has. But in the context 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# of a program, it is useful to have an enumeration. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ALL_TYPES = ( 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TOP_LEVEL, # The type of the 'Google Chrome' folder. 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APPS, 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_NOTIFICATION, 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_SETTINGS, 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL, 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL_PROFILE, 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BOOKMARK, 602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DEVICE_INFO, 612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DICTIONARY, 622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPERIMENTS, 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXTENSIONS, 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) HISTORY_DELETE_DIRECTIVE, 65c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) MANAGED_USER_SETTING, 6690dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) MANAGED_USER, 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NIGORI, 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PASSWORD, 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PREFERENCE, 70c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PRIORITY_PREFERENCE, 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SEARCH_ENGINE, 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SESSION, 732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SYNCED_NOTIFICATION, 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) THEME, 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TYPED_URL, 762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXTENSION_SETTINGS, 772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) FAVICON_IMAGES, 7890dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) FAVICON_TRACKING) = range(26) 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# An enumeration on the frequency at which the server should send errors 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# to the client. This would be specified by the url that triggers the error. 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Note: This enum should be kept in the same order as the enum in sync_test.h. 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SYNC_ERROR_FREQUENCY = ( 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_NONE, 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_ALWAYS, 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_TWO_THIRDS) = range(3) 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Well-known server tag of the top level 'Google Chrome' folder. 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)TOP_LEVEL_FOLDER_TAG = 'google_chrome' 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Given a sync type from ALL_TYPES, find the FieldDescriptor corresponding 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# to that datatype. Note that TOP_LEVEL has no such token. 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SYNC_TYPE_FIELDS = sync_pb2.EntitySpecifics.DESCRIPTOR.fields_by_name 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)SYNC_TYPE_TO_DESCRIPTOR = { 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_NOTIFICATION: SYNC_TYPE_FIELDS['app_notification'], 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APP_SETTINGS: SYNC_TYPE_FIELDS['app_setting'], 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) APPS: SYNC_TYPE_FIELDS['app'], 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL: SYNC_TYPE_FIELDS['autofill'], 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) AUTOFILL_PROFILE: SYNC_TYPE_FIELDS['autofill_profile'], 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) BOOKMARK: SYNC_TYPE_FIELDS['bookmark'], 1012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DEVICE_INFO: SYNC_TYPE_FIELDS['device_info'], 1022a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) DICTIONARY: SYNC_TYPE_FIELDS['dictionary'], 1032a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPERIMENTS: SYNC_TYPE_FIELDS['experiments'], 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXTENSION_SETTINGS: SYNC_TYPE_FIELDS['extension_setting'], 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) EXTENSIONS: SYNC_TYPE_FIELDS['extension'], 1062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) FAVICON_IMAGES: SYNC_TYPE_FIELDS['favicon_image'], 1072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) FAVICON_TRACKING: SYNC_TYPE_FIELDS['favicon_tracking'], 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) HISTORY_DELETE_DIRECTIVE: SYNC_TYPE_FIELDS['history_delete_directive'], 109c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) MANAGED_USER_SETTING: SYNC_TYPE_FIELDS['managed_user_setting'], 11090dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) MANAGED_USER: SYNC_TYPE_FIELDS['managed_user'], 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NIGORI: SYNC_TYPE_FIELDS['nigori'], 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PASSWORD: SYNC_TYPE_FIELDS['password'], 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PREFERENCE: SYNC_TYPE_FIELDS['preference'], 114c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PRIORITY_PREFERENCE: SYNC_TYPE_FIELDS['priority_preference'], 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SEARCH_ENGINE: SYNC_TYPE_FIELDS['search_engine'], 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SESSION: SYNC_TYPE_FIELDS['session'], 1172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SYNCED_NOTIFICATION: SYNC_TYPE_FIELDS["synced_notification"], 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) THEME: SYNC_TYPE_FIELDS['theme'], 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TYPED_URL: SYNC_TYPE_FIELDS['typed_url'], 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) } 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The parent ID used to indicate a top-level node. 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)ROOT_ID = '0' 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Unix time epoch in struct_time format. The tuple corresponds to UTC Wednesday 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Jan 1 1970, 00:00:00, non-dst. 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)UNIX_TIME_EPOCH = (1970, 1, 1, 0, 0, 0, 3, 1, 0) 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The number of characters in the server-generated encryption key. 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)KEYSTORE_KEY_LENGTH = 16 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# The hashed client tag for the keystore encryption experiment node. 1332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)KEYSTORE_ENCRYPTION_EXPERIMENT_TAG = "pis8ZRzh98/MKLtVEio2mr42LQA=" 1342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class Error(Exception): 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Error class for this module.""" 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class ProtobufDataTypeFieldNotUnique(Error): 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """An entry should not have more than one data type present.""" 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class DataTypeIdNotRecognized(Error): 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The requested data type is not recognized.""" 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class MigrationDoneError(Error): 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A server-side migration occurred; clients must re-sync some datatypes. 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Attributes: 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatypes: a list of the datatypes (python enum) needing migration. 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, datatypes): 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.datatypes = datatypes 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class StoreBirthdayError(Error): 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The client sent a birthday that doesn't correspond to this server.""" 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class TransientError(Error): 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The client would be sent a transient error.""" 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class SyncInducedError(Error): 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The client would be sent an error.""" 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class InducedErrorFrequencyNotDefined(Error): 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """The error frequency defined is not handled.""" 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetEntryType(entry): 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Extract the sync type from a SyncEntry. 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A SyncEntity protobuf object whose type to determine. 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) An enum value from ALL_TYPES if the entry's type can be determined, or None 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if the type cannot be determined. 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ProtobufDataTypeFieldNotUnique: More than one type was indicated by 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) the entry. 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.server_defined_unique_tag == TOP_LEVEL_FOLDER_TAG: 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return TOP_LEVEL 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry_types = GetEntryTypesFromSpecifics(entry.specifics) 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not entry_types: 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # If there is more than one, either there's a bug, or else the caller 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # should use GetEntryTypes. 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(entry_types) > 1: 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise ProtobufDataTypeFieldNotUnique 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return entry_types[0] 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetEntryTypesFromSpecifics(specifics): 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine the sync types indicated by an EntitySpecifics's field(s). 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) If the specifics have more than one recognized data type field (as commonly 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) happens with the requested_types field of GetUpdatesMessage), all types 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) will be returned. Callers must handle the possibility of the returned 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) value having more than one item. 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) specifics: A EntitySpecifics protobuf message whose extensions to 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) enumerate. 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A list of the sync types (values from ALL_TYPES) associated with each 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) recognized extension of the specifics message. 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return [data_type for data_type, field_descriptor 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) in SYNC_TYPE_TO_DESCRIPTOR.iteritems() 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if specifics.HasField(field_descriptor.name)] 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SyncTypeToProtocolDataTypeId(data_type): 2205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Convert from a sync type (python enum) to the protocol's data type id.""" 2215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SYNC_TYPE_TO_DESCRIPTOR[data_type].number 2225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ProtocolDataTypeIdToSyncType(protocol_data_type_id): 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Convert from the protocol's data type id to a sync type (python enum).""" 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, field_descriptor in SYNC_TYPE_TO_DESCRIPTOR.iteritems(): 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if field_descriptor.number == protocol_data_type_id: 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return data_type 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise DataTypeIdNotRecognized 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def DataTypeStringToSyncTypeLoose(data_type_string): 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Converts a human-readable string to a sync type (python enum). 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Capitalization and pluralization don't matter; this function is appropriate 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for values that might have been typed by a human being; e.g., command-line 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) flags or query parameters. 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type_string.isdigit(): 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ProtocolDataTypeIdToSyncType(int(data_type_string)) 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name = data_type_string.lower().rstrip('s') 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, field_descriptor in SYNC_TYPE_TO_DESCRIPTOR.iteritems(): 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if field_descriptor.name.lower().rstrip('s') == name: 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return data_type 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise DataTypeIdNotRecognized 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def MakeNewKeystoreKey(): 2492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Returns a new random keystore key.""" 2502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return ''.join(random.choice(string.ascii_uppercase + string.digits) 2512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for x in xrange(KEYSTORE_KEY_LENGTH)) 2522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SyncTypeToString(data_type): 2555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Formats a sync type enum (from ALL_TYPES) to a human-readable string.""" 2565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return SYNC_TYPE_TO_DESCRIPTOR[data_type].name 2575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def CallerInfoToString(caller_info_source): 2605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Formats a GetUpdatesSource enum value to a readable string.""" 2615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return get_updates_caller_info_pb2.GetUpdatesCallerInfo \ 2625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) .DESCRIPTOR.enum_types_by_name['GetUpdatesSource'] \ 2635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) .values_by_number[caller_info_source].name 2645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def ShortDatatypeListSummary(data_types): 2675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Formats compactly a list of sync types (python enums) for human eyes. 2685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This function is intended for use by logging. If the list of datatypes 2705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) contains almost all of the values, the return value will be expressed 2715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) in terms of the datatypes that aren't set. 2725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) included = set(data_types) - set([TOP_LEVEL]) 2745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not included: 2755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 'nothing' 2765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) excluded = set(ALL_TYPES) - included - set([TOP_LEVEL]) 2775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not excluded: 2785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 'everything' 2795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) simple_text = '+'.join(sorted([SyncTypeToString(x) for x in included])) 2805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) all_but_text = 'all except %s' % ( 2815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '+'.join(sorted([SyncTypeToString(x) for x in excluded]))) 2825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if len(included) < len(excluded) or len(simple_text) <= len(all_but_text): 2835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return simple_text 2845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 2855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return all_but_text 2865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetDefaultEntitySpecifics(data_type): 2895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Get an EntitySpecifics having a sync type's default field value.""" 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) specifics = sync_pb2.EntitySpecifics() 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type in SYNC_TYPE_TO_DESCRIPTOR: 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) descriptor = SYNC_TYPE_TO_DESCRIPTOR[data_type] 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) getattr(specifics, descriptor.name).SetInParent() 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return specifics 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class PermanentItem(object): 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A specification of one server-created permanent item. 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Attributes: 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tag: A known-to-the-client value that uniquely identifies a server-created 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) permanent item. 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name: The human-readable display name for this item. 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag: The tag of the permanent item's parent. If ROOT_ID, indicates 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) a top-level item. Otherwise, this must be the tag value of some other 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server-created permanent item. 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_type: A value from ALL_TYPES, giving the datatype of this permanent 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) item. This controls which types of client GetUpdates requests will 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cause the permanent item to be created and returned. 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) create_by_default: Whether the permanent item is created at startup or not. 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This value is set to True in the default case. Non-default permanent items 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) are those that are created only when a client explicitly tells the server 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to do so. 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, tag, name, parent_tag, sync_type, create_by_default=True): 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.tag = tag 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.name = name 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.parent_tag = parent_tag 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_type = sync_type 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.create_by_default = create_by_default 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class MigrationHistory(object): 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A record of the migration events associated with an account. 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Each migration event invalidates one or more datatypes on all clients 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) that had synced the datatype before the event. Such clients will continue 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to receive MigrationDone errors until they throw away their progress and 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) re-sync that datatype from the beginning. 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migrations = {} 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for datatype in ALL_TYPES: 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migrations[datatype] = [1] 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._next_migration_version = 2 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetLatestVersion(self, datatype): 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._migrations[datatype][-1] 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckAllCurrent(self, versions_map): 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises an error if any the provided versions are out of date. 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This function intentionally returns migrations in the order that they were 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) triggered. Doing it this way allows the client to queue up two migrations 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) in a row, so the second one is received while responding to the first. 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Arguments: 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) version_map: a map whose keys are datatypes and whose values are versions. 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MigrationDoneError: if a mismatch is found. 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) problems = {} 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for datatype, client_migration in versions_map.iteritems(): 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for server_migration in self._migrations[datatype]: 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if client_migration < server_migration: 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) problems.setdefault(server_migration, []).append(datatype) 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if problems: 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise MigrationDoneError(problems[min(problems.keys())]) 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def Bump(self, datatypes): 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Add a record of a migration, to cause errors on future requests.""" 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for idx, datatype in enumerate(datatypes): 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migrations[datatype].append(self._next_migration_version) 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._next_migration_version += 1 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class UpdateSieve(object): 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A filter to remove items the client has already seen.""" 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self, request, migration_history=None): 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._original_request = request 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state = {} 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_history = migration_history or MigrationHistory() 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check = {} 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if request.from_progress_marker: 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for marker in request.from_progress_marker: 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) data_type = ProtocolDataTypeIdToSyncType(marker.data_type_id) 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if marker.HasField('timestamp_token_for_migration'): 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp = marker.timestamp_token_for_migration 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if timestamp: 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check[data_type] = 1 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif marker.token: 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (timestamp, version) = pickle.loads(marker.token) 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check[data_type] = version 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif marker.HasField('token'): 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp = 0 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise ValueError('No timestamp information in progress marker.') 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) data_type = ProtocolDataTypeIdToSyncType(marker.data_type_id) 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state[data_type] = timestamp 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif request.HasField('from_timestamp'): 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type in GetEntryTypesFromSpecifics(request.requested_types): 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state[data_type] = request.from_timestamp 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_versions_to_check[data_type] = 1 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._state: 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._state[TOP_LEVEL] = min(self._state.itervalues()) 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def SummarizeRequest(self): 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamps = {} 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, timestamp in self._state.iteritems(): 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type == TOP_LEVEL: 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamps.setdefault(timestamp, []).append(data_type) 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ', '.join('<%s>@%d' % (ShortDatatypeListSummary(types), stamp) 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for stamp, types in sorted(timestamps.iteritems())) 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckMigrationState(self): 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._migration_history.CheckAllCurrent(self._migration_versions_to_check) 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ClientWantsItem(self, item): 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return true if the client hasn't already seen an item.""" 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._state.get(GetEntryType(item), sys.maxint) < item.version 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HasAnyTimestamp(self): 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return true if at least one datatype was requested.""" 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return bool(self._state) 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetMinTimestamp(self): 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return true the smallest timestamp requested across all datatypes.""" 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return min(self._state.itervalues()) 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetFirstTimeTypes(self): 4245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Return a list of datatypes requesting updates from timestamp zero.""" 4255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return [datatype for datatype, timestamp in self._state.iteritems() 4265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if timestamp == 0] 4275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def GetCreateMobileBookmarks(self): 4292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Return true if the client has requested to create the 'Mobile Bookmarks' 4302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) folder. 4312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 4322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return (self._original_request.HasField('create_mobile_bookmarks_folder') 4332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) and self._original_request.create_mobile_bookmarks_folder) 4342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def SaveProgress(self, new_timestamp, get_updates_response): 4365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Write the new_timestamp or new_progress_marker fields to a response.""" 4375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._original_request.from_progress_marker: 4385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for data_type, old_timestamp in self._state.iteritems(): 4395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if data_type == TOP_LEVEL: 4405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 4415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_marker = sync_pb2.DataTypeProgressMarker() 4425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_marker.data_type_id = SyncTypeToProtocolDataTypeId(data_type) 4435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) final_stamp = max(old_timestamp, new_timestamp) 4445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) final_migration = self._migration_history.GetLatestVersion(data_type) 4455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_marker.token = pickle.dumps((final_stamp, final_migration)) 4465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if new_marker not in self._original_request.from_progress_marker: 4475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) get_updates_response.new_progress_marker.add().MergeFrom(new_marker) 4485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif self._original_request.HasField('from_timestamp'): 4495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._original_request.from_timestamp < new_timestamp: 4505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) get_updates_response.new_timestamp = new_timestamp 4515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class SyncDataModel(object): 4545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Models the account state of one sync user.""" 4555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _BATCH_SIZE = 100 4565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Specify all the permanent items that a model might need. 4585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) _PERMANENT_ITEM_SPECS = [ 4592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_apps', name='Apps', 4602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=APPS), 4612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_app_notifications', name='App Notifications', 4622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=APP_NOTIFICATION), 4632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_app_settings', 4642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='App Settings', 4652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=APP_SETTINGS), 4665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_bookmarks', name='Bookmarks', 4672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=BOOKMARK), 4685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('bookmark_bar', name='Bookmark Bar', 4695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag='google_chrome_bookmarks', sync_type=BOOKMARK), 4705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('other_bookmarks', name='Other Bookmarks', 4715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag='google_chrome_bookmarks', sync_type=BOOKMARK), 4725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('synced_bookmarks', name='Synced Bookmarks', 4735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_tag='google_chrome_bookmarks', sync_type=BOOKMARK, 4745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) create_by_default=False), 4755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_autofill', name='Autofill', 4762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=AUTOFILL), 4775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_autofill_profiles', name='Autofill Profiles', 4782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=AUTOFILL_PROFILE), 4792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_device_info', name='Device Info', 4802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=DEVICE_INFO), 4812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_experiments', name='Experiments', 4822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=EXPERIMENTS), 4835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_extension_settings', 4845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name='Extension Settings', 4852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=EXTENSION_SETTINGS), 4865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_extensions', name='Extensions', 4872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=EXTENSIONS), 4885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_history_delete_directives', 4895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) name='History Delete Directives', 4902a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, 4915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_type=HISTORY_DELETE_DIRECTIVE), 4922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_favicon_images', 4932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='Favicon Images', 4942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, 4952a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sync_type=FAVICON_IMAGES), 4962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_favicon_tracking', 4972a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='Favicon Tracking', 4982a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, 4992a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sync_type=FAVICON_TRACKING), 500c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PermanentItem('google_chrome_managed_user_settings', 501c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) name='Managed User Settings', 502c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=MANAGED_USER_SETTING), 50390dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) PermanentItem('google_chrome_managed_users', 50490dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) name='Managed Users', 50590dce4d38c5ff5333bea97d859d4e484e27edf0cTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=MANAGED_USER), 5062a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_nigori', name='Nigori', 5072a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=NIGORI), 5085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_passwords', name='Passwords', 5092a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=PASSWORD), 5102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_preferences', name='Preferences', 5112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=PREFERENCE), 512c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) PermanentItem('google_chrome_priority_preferences', 513c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) name='Priority Preferences', 514c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=PRIORITY_PREFERENCE), 5152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_synced_notifications', 5162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) name='Synced Notifications', 5172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=SYNCED_NOTIFICATION), 5185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_search_engines', name='Search Engines', 5192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=SEARCH_ENGINE), 5205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_sessions', name='Sessions', 5212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=SESSION), 5225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_themes', name='Themes', 5232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=THEME), 5245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) PermanentItem('google_chrome_typed_urls', name='Typed URLs', 5252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=TYPED_URL), 5262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) PermanentItem('google_chrome_dictionary', name='Dictionary', 5272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parent_tag=ROOT_ID, sync_type=DICTIONARY), 5285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ] 5295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 5315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Monotonically increasing version number. The next object change will 5325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # take on this value + 1. 5335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._version = 0 5345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The definitive copy of this client's items: a map from ID string to a 5365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # SyncEntity protocol buffer. 5375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries = {} 5385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.ResetStoreBirthday() 5405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.migration_history = MigrationHistory() 5425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error = sync_pb2.ClientToServerResponse.Error() 5445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error_frequency = 0 5455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count_before_errors = 0 5465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._keys = [MakeNewKeystoreKey()] 5485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _SaveEntry(self, entry): 5505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Insert or update an entry in the change log, and give it a new version. 5515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The ID fields of this entry are assumed to be valid server IDs. This 5535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry will be updated with a new version number and sync_timestamp. 5545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 5565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: The entry to be added or updated. 5575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 5585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._version += 1 5595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Maintain a global (rather than per-item) sequence number and use it 5605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # both as the per-entry version as well as the update-progress timestamp. 5615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This simulates the behavior of the original server implementation. 5625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.version = self._version 5635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.sync_timestamp = self._version 5645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Preserve the originator info, which the client is not required to send 5665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # when updating. 5675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base_entry = self._entries.get(entry.id_string) 5685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if base_entry: 5695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_cache_guid = base_entry.originator_cache_guid 5705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_client_item_id = base_entry.originator_client_item_id 5715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string] = copy.deepcopy(entry) 5735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ServerTagToId(self, tag): 5755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine the server ID from a server-unique tag. 5765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting value is guaranteed not to collide with the other ID 5785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generation methods. 5795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 5815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype: The sync type (python enum) of the identified object. 5825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tag: The unique, known-to-the-client tag of a server-generated item. 5835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 5845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The string value of the computed server ID. 5855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 5865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not tag or tag == ROOT_ID: 5875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tag 5885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) spec = [x for x in self._PERMANENT_ITEM_SPECS if x.tag == tag][0] 5895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(spec.sync_type, '<server tag>%s' % tag) 5905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ClientTagToId(self, datatype, tag): 5925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine the server ID from a client-unique tag. 5935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting value is guaranteed not to collide with the other ID 5955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generation methods. 5965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 5975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 5985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype: The sync type (python enum) of the identified object. 5995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tag: The unique, opaque-to-the-server tag of a client-tagged item. 6005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 6015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The string value of the computed server ID. 6025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(datatype, '<client tag>%s' % tag) 6045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ClientIdToId(self, datatype, client_guid, client_item_id): 6065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Compute a unique server ID from a client-local ID tag. 6075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting value is guaranteed not to collide with the other ID 6095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generation methods. 6105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype: The sync type (python enum) of the identified object. 6135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_guid: A globally unique ID that identifies the client which 6145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) created this item. 6155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_item_id: An ID that uniquely identifies this item on the client 6165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) which created it. 6175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 6185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The string value of the computed server ID. 6195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Using the client ID info is not required here (we could instead generate 6215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # a random ID), but it's useful for debugging. 6225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(datatype, 6235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<server ID originally>%s/%s' % (client_guid, client_item_id)) 6245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _MakeCurrentId(self, datatype, inner_id): 6265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return '%d^%d^%s' % (datatype, 6275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.migration_history.GetLatestVersion(datatype), 6285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) inner_id) 6295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ExtractIdInfo(self, id_string): 6315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not id_string or id_string == ROOT_ID: 6325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 6335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype_string, separator, remainder = id_string.partition('^') 6345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) migration_version_string, separator, inner_id = remainder.partition('^') 6355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (int(datatype_string), int(migration_version_string), inner_id) 6365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _WritePosition(self, entry, parent_id): 6385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Ensure the entry has an absolute, numeric position and parent_id. 6395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Historically, clients would specify positions using the predecessor-based 6415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) references in the insert_after_item_id field; starting July 2011, this 6425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) was changed and Chrome now sends up the absolute position. The server 6435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) must store a position_in_parent value and must not maintain 6445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) insert_after_item_id. 645c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) Starting in Jan 2013, the client will also send up a unique_position field 646c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) which should be saved and returned on subsequent GetUpdates. 6475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: The entry for which to write a position. Its ID field are 650c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) assumed to be server IDs. This entry will have its parent_id_string, 651c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) position_in_parent and unique_position fields updated; its 652c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) insert_after_item_id field will be cleared. 6535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parent_id: The ID of the entry intended as the new parent. 6545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string = parent_id 6575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not entry.HasField('position_in_parent'): 6585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.position_in_parent = 1337 # A debuggable, distinctive default. 6595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.ClearField('insert_after_item_id') 6605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _ItemExists(self, id_string): 6625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Determine whether an item exists in the changelog.""" 6635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return id_string in self._entries 6645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CreatePermanentItem(self, spec): 6665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Create one permanent item from its spec, if it doesn't exist. 6675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting item is added to the changelog. 6695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) spec: A PermanentItem object holding the properties of the item to create. 6725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) id_string = self._ServerTagToId(spec.tag) 6745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._ItemExists(id_string): 6755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 6765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Creating permanent item: %s' % spec.name 6775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry = sync_pb2.SyncEntity() 6785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.id_string = id_string 6795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.non_unique_name = spec.name 6805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.name = spec.name 6815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.server_defined_unique_tag = spec.tag 6825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.folder = True 6835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.deleted = False 6845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.specifics.CopyFrom(GetDefaultEntitySpecifics(spec.sync_type)) 6855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._WritePosition(entry, self._ServerTagToId(spec.parent_tag)) 6865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(entry) 6875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CreateDefaultPermanentItems(self, requested_types): 6895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Ensure creation of all default permanent items for a given set of types. 6905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 6925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) requested_types: A list of sync data types from ALL_TYPES. 6935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) All default permanent items of only these types will be created. 6945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 6955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for spec in self._PERMANENT_ITEM_SPECS: 6965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if spec.sync_type in requested_types and spec.create_by_default: 6975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._CreatePermanentItem(spec) 6985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 6995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ResetStoreBirthday(self): 7005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Resets the store birthday to a random value.""" 7015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): uuid.uuid1() is better, but python 2.5 only. 7025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.store_birthday = '%0.30f' % random.random() 7035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def StoreBirthday(self): 7055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Gets the store birthday.""" 7065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.store_birthday 7075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetChanges(self, sieve): 7095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Get entries which have changed, oldest first. 7105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The returned entries are limited to being _BATCH_SIZE many. The entries 7125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) are returned in strict version order. 7135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sieve: An update sieve to use to filter out updates the client 7165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) has already seen. 7175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 7185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A tuple of (version, entries, changes_remaining). Version is a new 7195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp value, which should be used as the starting point for the 7205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) next query. Entries is the batch of entries meeting the current 7215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) timestamp query. Changes_remaining indicates the number of changes 7225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) left on the server after this batch. 7235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not sieve.HasAnyTimestamp(): 7255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (0, [], 0) 7265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) min_timestamp = sieve.GetMinTimestamp() 7272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) first_time_types = sieve.GetFirstTimeTypes() 7282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._CreateDefaultPermanentItems(first_time_types) 7292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Mobile bookmark folder is not created by default, create it only when 7302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # client requested it. 7312a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if (sieve.GetCreateMobileBookmarks() and 7322a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) first_time_types.count(BOOKMARK) > 0): 7332a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.TriggerCreateSyncedBookmarks() 7342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) change_log = sorted(self._entries.values(), 7365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) key=operator.attrgetter('version')) 7375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_changes = [x for x in change_log if x.version > min_timestamp] 7385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Pick batch_size new changes, and then filter them. This matches 7395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # the RPC behavior of the production sync server. 7405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) batch = new_changes[:self._BATCH_SIZE] 7415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not batch: 7425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Client is up to date. 7435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (min_timestamp, [], 0) 7445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Restrict batch to requested types. Tombstones are untyped 7465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # and will always get included. 7475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) filtered = [copy.deepcopy(item) for item in batch 7485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if item.deleted or sieve.ClientWantsItem(item)] 7495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The new client timestamp is the timestamp of the last item in the 7515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # batch, even if that item was filtered out. 7525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (batch[-1].version, filtered, len(new_changes) - len(batch)) 7535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def GetKeystoreKeys(self): 7552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Returns the encryption keys for this account.""" 7562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) print "Returning encryption keys: %s" % self._keys 7572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return self._keys 7585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CopyOverImmutableFields(self, entry): 7605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Preserve immutable fields by copying pre-commit state. 7615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A sync entity from the client. 7645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.id_string in self._entries: 7665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._entries[entry.id_string].HasField( 7675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'server_defined_unique_tag'): 7685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.server_defined_unique_tag = ( 7695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string].server_defined_unique_tag) 7705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CheckVersionForCommit(self, entry): 7725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Perform an optimistic concurrency check on the version number. 7735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Clients are only allowed to commit if they report having seen the most 7755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) recent version of an object. 7765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A sync entity from the client. It is assumed that ID fields 7795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) have been converted to server IDs. 7805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 7815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A boolean value indicating whether the client's version matches the 7825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) newest server version for the given entry. 7835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 7845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.id_string in self._entries: 7855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Allow edits/deletes if the version matches, and any undeletion. 7865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (self._entries[entry.id_string].version == entry.version or 7875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string].deleted) 7885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 7895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Allow unknown ID only if the client thinks it's new too. 7905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return entry.version == 0 7915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _CheckParentIdForCommit(self, entry): 7935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Check that the parent ID referenced in a SyncEntity actually exists. 7945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 7955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 7965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A sync entity from the client. It is assumed that ID fields 7975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) have been converted to server IDs. 7985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 7995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A boolean value indicating whether the entity's parent ID is an object 8005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) that actually exists (and is not deleted) in the current account state. 8015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string == ROOT_ID: 8035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This is generally allowed. 8045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 8055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string not in self._entries: 8065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Warning: Client sent unknown ID. Should never happen.' 8075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string == entry.id_string: 8095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Warning: Client sent circular reference. Should never happen.' 8105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._entries[entry.parent_id_string].deleted: 8125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This can happen in a race condition between two clients. 8135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self._entries[entry.parent_id_string].folder: 8155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Warning: Client sent non-folder parent. Should never happen.' 8165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 8175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 8185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _RewriteIdsAsServerIds(self, entry, cache_guid, commit_session): 8205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Convert ID fields in a client sync entry to server IDs. 8215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A commit batch sent by a client may contain new items for which the 8235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server has not generated IDs yet. And within a commit batch, later 8245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) items are allowed to refer to earlier items. This method will 8255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) generate server IDs for new items, as well as rewrite references 8265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to items whose server IDs were generated earlier in the batch. 8275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 8295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: The client sync entry to modify. 8305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cache_guid: The globally unique ID of the client that sent this 8315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit request. 8325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_session: A dictionary mapping the original IDs to the new server 8335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) IDs, for any items committed earlier in the batch. 8345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.version == 0: 8365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) data_type = GetEntryType(entry) 8375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.HasField('client_defined_unique_tag'): 8385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # When present, this should determine the item's ID. 8395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_id = self._ClientTagToId(data_type, entry.client_defined_unique_tag) 8405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_id = self._ClientIdToId(data_type, cache_guid, entry.id_string) 8425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_cache_guid = cache_guid 8435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_client_item_id = entry.id_string 8445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_session[entry.id_string] = new_id # Remember the remapping. 8455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.id_string = new_id 8465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string in commit_session: 8475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string = commit_session[entry.parent_id_string] 8485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.insert_after_item_id in commit_session: 8495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.insert_after_item_id = commit_session[entry.insert_after_item_id] 8505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ValidateCommitEntries(self, entries): 8525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raise an exception if a commit batch contains any global errors. 8535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Arguments: 8555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entries: an iterable containing commit-form SyncEntity protocol buffers. 8565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 8585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) MigrationDoneError: if any of the entries reference a recently-migrated 8595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype. 8605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_ids_in_commit = set() 8625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) local_ids_in_commit = set() 8635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in entries: 8645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.version: 8655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_ids_in_commit.add(entry.id_string) 8665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 8675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) local_ids_in_commit.add(entry.id_string) 8685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.HasField('parent_id_string'): 8695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.parent_id_string not in local_ids_in_commit: 8705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_ids_in_commit.add(entry.parent_id_string) 8715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) versions_present = {} 8735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for server_id in server_ids_in_commit: 8745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed = self._ExtractIdInfo(server_id) 8755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if parsed: 8765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype, version, _ = parsed 8775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) versions_present.setdefault(datatype, []).append(version) 8785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.migration_history.CheckAllCurrent( 8805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) dict((k, min(v)) for k, v in versions_present.iteritems())) 8815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CommitEntry(self, entry, cache_guid, commit_session): 8835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Attempt to commit one entry to the user's account. 8845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 8865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry: A SyncEntity protobuf representing desired object changes. 8875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) cache_guid: A string value uniquely identifying the client; this 8885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) is used for ID generation and will determine the originator_cache_guid 8895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if the entry is new. 8905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_session: A dictionary mapping client IDs to server IDs for any 8915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) objects committed earlier this session. If the entry gets a new ID 8925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) during commit, the change will be recorded here. 8935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 8945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A SyncEntity reflecting the post-commit value of the entry, or None 8955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if the entry was not committed due to an error. 8965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 8975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry = copy.deepcopy(entry) 8985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 8995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Generate server IDs for this entry, and write generated server IDs 9005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # from earlier entries into the message's fields, as appropriate. The 9015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # ID generation state is stored in 'commit_session'. 9025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._RewriteIdsAsServerIds(entry, cache_guid, commit_session) 9035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Perform the optimistic concurrency check on the entry's version number. 9055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Clients are not allowed to commit unless they indicate that they've seen 9065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # the most recent version of an object. 9075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self._CheckVersionForCommit(entry): 9085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 9095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Check the validity of the parent ID; it must exist at this point. 9115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): Implement cycle detection and resolution. 9125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not self._CheckParentIdForCommit(entry): 9135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 9145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._CopyOverImmutableFields(entry); 9165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # At this point, the commit is definitely going to happen. 9185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Deletion works by storing a limited record for an entry, called a 9205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # tombstone. A sync server must track deleted IDs forever, since it does 9215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # not keep track of client knowledge (there's no deletion ACK event). 9225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.deleted: 9235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def MakeTombstone(id_string): 9245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Make a tombstone entry that will replace the entry being deleted. 9255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 9275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) id_string: Index of the SyncEntity to be deleted. 9285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 9295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A new SyncEntity reflecting the fact that the entry is deleted. 9305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 9315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Only the ID, version and deletion state are preserved on a tombstone. 9325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): Does the production server not preserve the type? Not 9335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # doing so means that tombstones cannot be filtered based on 9345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # requested_types at GetUpdates time. 9355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone = sync_pb2.SyncEntity() 9365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone.id_string = id_string 9375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone.deleted = True 9385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tombstone.name = '' 9395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tombstone 9405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def IsChild(child_id): 9425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Check if a SyncEntity is a child of entry, or any of its children. 9435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 9455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) child_id: Index of the SyncEntity that is a possible child of entry. 9465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 9475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) True if it is a child; false otherwise. 9485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 9495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if child_id not in self._entries: 9505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return False 9515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self._entries[child_id].parent_id_string == entry.id_string: 9525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return True 9535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return IsChild(self._entries[child_id].parent_id_string) 9545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Identify any children entry might have. 9565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) child_ids = [child.id_string for child in self._entries.itervalues() 9575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if IsChild(child.id_string)] 9585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Mark all children that were identified as deleted. 9605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for child_id in child_ids: 9615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(MakeTombstone(child_id)) 9625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Delete entry itself. 9645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry = MakeTombstone(entry.id_string) 9655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 9665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Comments in sync.proto detail how the representation of positional 967c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # ordering works. 968c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 969c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # We've almost fully deprecated the 'insert_after_item_id' field. 970c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # The 'position_in_parent' field is also deprecated, but as of Jan 2013 971c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # is still in common use. The 'unique_position' field is the latest 972c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # and greatest in positioning technology. 973c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # 974c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) # This server supports 'position_in_parent' and 'unique_position'. 9755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._WritePosition(entry, entry.parent_id_string) 9765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Preserve the originator info, which the client is not required to send 9785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # when updating. 9795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) base_entry = self._entries.get(entry.id_string) 9805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if base_entry and not entry.HasField('originator_cache_guid'): 9815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_cache_guid = base_entry.originator_cache_guid 9825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.originator_client_item_id = base_entry.originator_client_item_id 9835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Store the current time since the Unix epoch in milliseconds. 9855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.mtime = (int((time.mktime(time.gmtime()) - 9865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) time.mktime(UNIX_TIME_EPOCH))*1000)) 9875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Commit the change. This also updates the version number. 9895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(entry) 9905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return entry 9915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 9925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def _RewriteVersionInId(self, id_string): 9935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Rewrites an ID so that its migration version becomes current.""" 9945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed_id = self._ExtractIdInfo(id_string) 9955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not parsed_id: 9965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return id_string 9975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatype, old_migration_version, inner_id = parsed_id 9985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self._MakeCurrentId(datatype, inner_id) 9995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def TriggerMigration(self, datatypes): 10015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Cause a migration to occur for a set of datatypes on this account. 10025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Clients will see the MIGRATION_DONE error for these datatypes until they 10045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) resync them. 10055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 10065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) versions_to_remap = self.migration_history.Bump(datatypes) 10075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) all_entries = self._entries.values() 10085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries.clear() 10095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in all_entries: 10105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_id = self._RewriteVersionInId(entry.id_string) 10115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.id_string = new_id 10125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if entry.HasField('parent_id_string'): 10135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string = self._RewriteVersionInId( 10145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) entry.parent_id_string) 10155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._entries[entry.id_string] = entry 10165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def TriggerSyncTabFavicons(self): 10185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Set the 'sync_tab_favicons' field to this account's nigori node. 10195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) If the field is not currently set, will write a new nigori node entry 10215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) with the field set. Else does nothing. 10225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 10235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_tag = "google_chrome_nigori" 10255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_original = self._entries.get(self._ServerTagToId(nigori_tag)) 10265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (nigori_original.specifics.nigori.sync_tab_favicons): 10275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 10285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_new = copy.deepcopy(nigori_original) 10295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) nigori_new.specifics.nigori.sync_tabs = True 10305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._SaveEntry(nigori_new) 10315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def TriggerCreateSyncedBookmarks(self): 10335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Create the Synced Bookmarks folder under the Bookmarks permanent item. 10345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Clients will then receive the Synced Bookmarks folder on future 10365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) GetUpdates, and new bookmarks can be added within the Synced Bookmarks 10375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) folder. 10385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 10395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) synced_bookmarks_spec, = [spec for spec in self._PERMANENT_ITEM_SPECS 10415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if spec.name == "Synced Bookmarks"] 10425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self._CreatePermanentItem(synced_bookmarks_spec) 10435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def TriggerEnableKeystoreEncryption(self): 10452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Create the keystore_encryption experiment entity and enable it. 10462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) A new entity within the EXPERIMENTS datatype is created with the unique 10482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) client tag "keystore_encryption" if it doesn't already exist. The 10492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_encryption message is then filled with |enabled| set to true. 10502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 10512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) experiment_id = self._ServerTagToId("google_chrome_experiments") 10532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_encryption_id = self._ClientTagToId( 10542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) EXPERIMENTS, 10552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) KEYSTORE_ENCRYPTION_EXPERIMENT_TAG) 10562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry = self._entries.get(keystore_encryption_id) 10572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if keystore_entry is None: 10582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry = sync_pb2.SyncEntity() 10592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.id_string = keystore_encryption_id 10602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.name = "Keystore Encryption" 10612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.client_defined_unique_tag = ( 10622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) KEYSTORE_ENCRYPTION_EXPERIMENT_TAG) 10632a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.folder = False 10642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.deleted = False 10652a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.specifics.CopyFrom(GetDefaultEntitySpecifics(EXPERIMENTS)) 10662a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._WritePosition(keystore_entry, experiment_id) 10672a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) keystore_entry.specifics.experiments.keystore_encryption.enabled = True 10692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._SaveEntry(keystore_entry) 10712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def TriggerRotateKeystoreKeys(self): 10732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Rotate the current set of keystore encryption keys. 10742a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10752a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) |self._keys| will have a new random encryption key appended to it. We touch 10762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) the nigori node so that each client will receive the new encryption keys 10772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) only once. 10782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 10792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Add a new encryption key. 10812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._keys += [MakeNewKeystoreKey(), ] 10822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Increment the nigori node's timestamp, so clients will get the new keys 10842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # on their next GetUpdates (any time the nigori node is sent back, we also 10852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # send back the keystore keys). 10862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) nigori_tag = "google_chrome_nigori" 10872a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self._SaveEntry(self._entries.get(self._ServerTagToId(nigori_tag))) 10882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 10895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def SetInducedError(self, error, error_frequency, 10905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_count_before_errors): 10915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error = error 10925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.induced_error_frequency = error_frequency 10935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count_before_errors = sync_count_before_errors 10945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetInducedError(self): 10965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.induced_error 10975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 10995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class TestServer(object): 11005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """An object to handle requests for one (and only one) Chrome Sync account. 11015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) TestServer consumes the sync command messages that are the outermost 11035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) layers of the protocol, performs the corresponding actions on its 11042a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) SyncDataModel, and constructs an appropriate response message. 11055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 11065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def __init__(self): 11085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # The implementation supports exactly one account; its state is here. 11095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account = SyncDataModel() 11105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock = threading.Lock() 11115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Clients that have talked to us: a map from the full client ID 11125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # to its nickname. 11135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.clients = {} 11145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.client_name_generator = ('+' * times + chr(c) 11155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for times in xrange(0, sys.maxint) for c in xrange(ord('A'), ord('Z'))) 11165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.transient_error = False 11175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count = 0 11185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def GetShortClientName(self, query): 11205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed = cgi.parse_qs(query[query.find('?')+1:]) 11215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_id = parsed.get('client_id') 11225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not client_id: 11235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return '?' 11245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) client_id = client_id[0] 11255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if client_id not in self.clients: 11265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.clients[client_id] = self.client_name_generator.next() 11275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return self.clients[client_id] 11285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckStoreBirthday(self, request): 11305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises StoreBirthdayError if the request's birthday is a mismatch.""" 11315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not request.HasField('store_birthday'): 11325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return 11335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.account.StoreBirthday() != request.store_birthday: 11345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise StoreBirthdayError 11355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckTransientError(self): 11375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises TransientError if transient_error variable is set.""" 11385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.transient_error: 11395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise TransientError 11405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def CheckSendError(self): 11425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Raises SyncInducedError if needed.""" 11435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (self.account.induced_error.error_type != 11445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) sync_enums_pb2.SyncEnums.UNKNOWN): 11455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Always means return the given error for all requests. 11465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if self.account.induced_error_frequency == ERROR_FREQUENCY_ALWAYS: 11475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise SyncInducedError 11485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # This means the FIRST 2 requests of every 3 requests 11495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # return an error. Don't switch the order of failures. There are 11505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # test cases that rely on the first 2 being the failure rather than 11515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # the last 2. 11525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif (self.account.induced_error_frequency == 11535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ERROR_FREQUENCY_TWO_THIRDS): 11545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (((self.sync_count - 11555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.sync_count_before_errors) % 3) != 0): 11565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise SyncInducedError 11575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise InducedErrorFrequencyNotDefined 11595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleMigrate(self, path): 11615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) query = urlparse.urlparse(path)[4] 11625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 200 11635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.acquire() 11645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 11655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) datatypes = [DataTypeStringToSyncTypeLoose(x) 11665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for x in urlparse.parse_qs(query).get('type',[])] 11675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if datatypes: 11685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.TriggerMigration(datatypes) 11695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Migrated datatypes %s' % ( 11705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ' and '.join(SyncTypeToString(x).upper() for x in datatypes)) 11715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 11725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Please specify one or more <i>type=name</i> parameters' 11735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 400 11745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except DataTypeIdNotRecognized, error: 11755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Could not interpret datatype name' 11765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 400 11775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 11785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.release() 11795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (code, '<html><title>Migration: %d</title><H1>%d %s</H1></html>' % 11805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (code, code, response)) 11815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleSetInducedError(self, path): 11835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) query = urlparse.urlparse(path)[4] 11845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.acquire() 11855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 200 11865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Success' 11875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error = sync_pb2.ClientToServerResponse.Error() 11885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 11895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error_type = urlparse.parse_qs(query)['error'] 11905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) action = urlparse.parse_qs(query)['action'] 11915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_type = int(error_type[0]) 11925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.action = int(action[0]) 11935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 11945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.url = (urlparse.parse_qs(query)['url'])[0] 11955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 11965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.url = '' 11975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 11985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_description =( 11995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (urlparse.parse_qs(query)['error_description'])[0]) 12005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 12015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_description = '' 12025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error_frequency = int((urlparse.parse_qs(query)['frequency'])[0]) 12045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except KeyError: 12055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error_frequency = ERROR_FREQUENCY_ALWAYS 12065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.SetInducedError(error, error_frequency, self.sync_count) 12075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = ('Error = %d, action = %d, url = %s, description = %s' % 12085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (error.error_type, error.action, 12095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.url, 12105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error.error_description)) 12115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except error: 12125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = 'Could not parse url' 12135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) code = 400 12145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 12155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.release() 12165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (code, '<html><title>SetError: %d</title><H1>%d %s</H1></html>' % 12175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) (code, code, response)) 12185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCreateBirthdayError(self): 12205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.ResetStoreBirthday() 12215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Birthday error</title><H1>Birthday error</H1></html>') 12245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleSetTransientError(self): 12265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.transient_error = True 12275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Transient error</title><H1>Transient error</H1></html>') 12305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleSetSyncTabFavicons(self): 12325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Set 'sync_tab_favicons' field of the nigori node for this account.""" 12335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.TriggerSyncTabFavicons() 12345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Tab Favicons</title><H1>Tab Favicons</H1></html>') 12375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCreateSyncedBookmarks(self): 12395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Create the Synced Bookmarks folder under Bookmarks.""" 12405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.TriggerCreateSyncedBookmarks() 12415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return ( 12425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 200, 12435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) '<html><title>Synced Bookmarks</title><H1>Synced Bookmarks</H1></html>') 12445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def HandleEnableKeystoreEncryption(self): 12462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Enables the keystore encryption experiment.""" 12472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.account.TriggerEnableKeystoreEncryption() 12482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return ( 12492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 200, 12502a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<html><title>Enable Keystore Encryption</title>' 12512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<H1>Enable Keystore Encryption</H1></html>') 12522a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 12532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) def HandleRotateKeystoreKeys(self): 12542a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Rotate the keystore encryption keys.""" 12552a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) self.account.TriggerRotateKeystoreKeys() 12562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) return ( 12572a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 200, 12582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<html><title>Rotate Keystore Keys</title>' 12592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '<H1>Rotate Keystore Keys</H1></html>') 12602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 12615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCommand(self, query, raw_request): 12625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Decode and handle a sync command from a raw input of bytes. 12635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This is the main entry point for this class. It is safe to call this 12655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) method from multiple threads. 12665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 12685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raw_request: An iterable byte sequence to be interpreted as a sync 12695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) protocol command. 12705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 12715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A tuple (response_code, raw_response); the first value is an HTTP 12725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) result code, while the second value is a string of bytes which is the 12735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) serialized reply to the command. 12745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 12755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.acquire() 12765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.sync_count += 1 12775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def print_context(direction): 12785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '[Client %s %s %s.py]' % (self.GetShortClientName(query), direction, 12795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) __name__), 12805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) try: 12825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request = sync_pb2.ClientToServerMessage() 12835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) request.MergeFromString(raw_request) 12845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) contents = request.message_contents 12855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = sync_pb2.ClientToServerResponse() 12875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.SUCCESS 12885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.CheckStoreBirthday(request) 12895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 12905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.CheckTransientError() 12915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.CheckSendError() 12925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('->') 12945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 12955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if contents == sync_pb2.ClientToServerMessage.AUTHENTICATE: 12965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Authenticate' 12975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # We accept any authentication token, and support only one account. 12985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(nick): Mock out the GAIA authentication as well; hook up here. 12995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.authenticate.user.email = 'syncjuser@chromium' 13005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.authenticate.user.display_name = 'Sync J User' 13015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif contents == sync_pb2.ClientToServerMessage.COMMIT: 13025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Commit %d item(s)' % len(request.commit.entries) 13035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.HandleCommit(request.commit, response.commit) 13045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) elif contents == sync_pb2.ClientToServerMessage.GET_UPDATES: 13055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'GetUpdates', 13065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.HandleGetUpdates(request.get_updates, response.get_updates) 13075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print '%d update(s)' % len(response.get_updates.entries) 13095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 13105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'Unrecognizable sync request!' 13115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (400, None) # Bad request. 13125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except MigrationDoneError, error: 13145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'MIGRATION_DONE: <%s>' % (ShortDatatypeListSummary(error.datatypes)) 13165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = sync_pb2.ClientToServerResponse() 13175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.MIGRATION_DONE 13195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.migrated_data_type_id[:] = [ 13205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) SyncTypeToProtocolDataTypeId(x) for x in error.datatypes] 13215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except StoreBirthdayError, error: 13235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'NOT_MY_BIRTHDAY' 13255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response = sync_pb2.ClientToServerResponse() 13265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.NOT_MY_BIRTHDAY 13285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except TransientError, error: 13305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) ### This is deprecated now. Would be removed once test cases are removed. 13315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'TRANSIENT_ERROR' 13335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error_code = sync_enums_pb2.SyncEnums.TRANSIENT_ERROR 13355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) except SyncInducedError, error: 13375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print_context('<-') 13385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print 'INDUCED_ERROR' 13395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.store_birthday = self.account.store_birthday 13405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) error = self.account.GetInducedError() 13415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.error_type = error.error_type 13425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.url = error.url 13435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.error_description = error.error_description 13445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) response.error.action = error.action 13455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return (200, response.SerializeToString()) 13465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) finally: 13475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account_lock.release() 13485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleCommit(self, commit_message, commit_response): 13505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Respond to a Commit request by updating the user's account state. 13515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Commit attempts stop after the first error, returning a CONFLICT result 13535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for any unattempted entries. 13545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 13565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_message: A sync_pb.CommitMessage protobuf holding the content 13575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) of the client's request. 13585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_response: A sync_pb.CommitResponse protobuf into which a reply 13595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to the client request will be written. 13605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 13615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) commit_response.SetInParent() 13625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) batch_failure = False 13635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) session = {} # Tracks ID renaming during the commit operation. 13645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) guid = commit_message.cache_guid 13655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.account.ValidateCommitEntries(commit_message.entries) 13675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in commit_message.entries: 13695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_entry = None 13705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not batch_failure: 13715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Try to commit the change to the account. 13725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) server_entry = self.account.CommitEntry(entry, guid, session) 13735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # An entryresponse is returned in both success and failure cases. 13755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply = commit_response.entryresponse.add() 13765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not server_entry: 13775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.response_type = sync_pb2.CommitResponse.CONFLICT 13785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.error_message = 'Conflict.' 13795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) batch_failure = True # One failure halts the batch. 13805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 13815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.response_type = sync_pb2.CommitResponse.SUCCESS 13825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # These are the properties that the server is allowed to override 13835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # during commit; the client wants to know their values at the end 13845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # of the operation. 13855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.id_string = server_entry.id_string 13865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not server_entry.deleted: 13875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Note: the production server doesn't actually send the 13885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # parent_id_string on commit responses, so we don't either. 13895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.position_in_parent = server_entry.position_in_parent 13905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.version = server_entry.version 13915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.name = server_entry.name 13925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.non_unique_name = server_entry.non_unique_name 13935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 13945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.version = entry.version + 1 13955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def HandleGetUpdates(self, update_request, update_response): 13975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Respond to a GetUpdates request by querying the user's account. 13985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 13995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 14005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_request: A sync_pb.GetUpdatesMessage protobuf holding the content 14015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) of the client's request. 14025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_response: A sync_pb.GetUpdatesResponse protobuf into which a reply 14035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) to the client request will be written. 14045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 14055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_response.SetInParent() 14065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_sieve = UpdateSieve(update_request, self.account.migration_history) 14075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print CallerInfoToString(update_request.caller_info.source), 14095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) print update_sieve.SummarizeRequest() 14105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_sieve.CheckMigrationState() 14125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_timestamp, entries, remaining = self.account.GetChanges(update_sieve) 14145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_response.changes_remaining = remaining 14162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sending_nigori_node = False 14175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for entry in entries: 14182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if entry.name == 'Nigori': 14192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sending_nigori_node = True 14205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply = update_response.entries.add() 14215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reply.CopyFrom(entry) 14225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) update_sieve.SaveProgress(new_timestamp, update_response) 14235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 14242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if update_request.need_encryption_key or sending_nigori_node: 14252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) update_response.encryption_keys.extend(self.account.GetKeystoreKeys()) 1426