1// Copyright (C) 2014 Google Inc. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15#include <libaddressinput/preload_supplier.h> 16 17#include <libaddressinput/address_data.h> 18#include <libaddressinput/address_field.h> 19#include <libaddressinput/callback.h> 20#include <libaddressinput/supplier.h> 21#include <libaddressinput/util/basictypes.h> 22#include <libaddressinput/util/scoped_ptr.h> 23 24#include <algorithm> 25#include <cassert> 26#include <cstddef> 27#include <functional> 28#include <map> 29#include <set> 30#include <stack> 31#include <string> 32#include <utility> 33#include <vector> 34 35#include "lookup_key.h" 36#include "region_data_constants.h" 37#include "retriever.h" 38#include "rule.h" 39#include "util/json.h" 40#include "util/string_compare.h" 41 42namespace i18n { 43namespace addressinput { 44 45namespace { 46 47// STL predicate less<> that uses StringCompare to match strings that a human 48// reader would consider to be "the same". The default implementation just does 49// case insensitive string comparison, but StringCompare can be overriden with 50// more sophisticated implementations. 51class IndexLess : public std::binary_function<std::string, std::string, bool> { 52 public: 53 result_type operator()(const first_argument_type& a, 54 const second_argument_type& b) const { 55 static const StringCompare kStringCompare; 56 return kStringCompare.NaturalLess(a, b); 57 } 58}; 59 60} // namespace 61 62class IndexMap : public std::map<std::string, const Rule*, IndexLess> {}; 63 64namespace { 65 66class Helper { 67 public: 68 // Does not take ownership of its parameters. 69 Helper(const std::string& region_code, 70 const std::string& key, 71 const PreloadSupplier::Callback& loaded, 72 const Retriever& retriever, 73 std::set<std::string>* pending, 74 IndexMap* rule_index, 75 std::vector<const Rule*>* rule_storage, 76 std::map<std::string, const Rule*>* region_rules) 77 : region_code_(region_code), 78 loaded_(loaded), 79 pending_(pending), 80 rule_index_(rule_index), 81 rule_storage_(rule_storage), 82 region_rules_(region_rules), 83 retrieved_(BuildCallback(this, &Helper::OnRetrieved)) { 84 assert(pending_ != NULL); 85 assert(rule_index_ != NULL); 86 assert(rule_storage_ != NULL); 87 assert(region_rules_ != NULL); 88 assert(retrieved_ != NULL); 89 pending_->insert(key); 90 retriever.Retrieve(key, *retrieved_); 91 } 92 93 private: 94 ~Helper() {} 95 96 void OnRetrieved(bool success, 97 const std::string& key, 98 const std::string& data) { 99 int rule_count = 0; 100 101 size_t status = pending_->erase(key); 102 assert(status == 1); // There will always be one item erased from the set. 103 (void)status; // Prevent unused variable if assert() is optimized away. 104 105 Json json; 106 std::string id; 107 std::vector<const Rule*> sub_rules; 108 109 IndexMap::iterator last_index_it = rule_index_->end(); 110 IndexMap::iterator last_latin_it = rule_index_->end(); 111 std::map<std::string, const Rule*>::iterator last_region_it = 112 region_rules_->end(); 113 114 IndexMap::const_iterator hints[arraysize(LookupKey::kHierarchy) - 1]; 115 std::fill(hints, hints + arraysize(hints), rule_index_->end()); 116 117 if (!success) { 118 goto callback; 119 } 120 121 if (!json.ParseObject(data)) { 122 success = false; 123 goto callback; 124 } 125 126 for (std::vector<const Json*>::const_iterator 127 it = json.GetSubDictionaries().begin(); 128 it != json.GetSubDictionaries().end(); 129 ++it) { 130 const Json* value = *it; 131 assert(value != NULL); 132 if (!value->GetStringValueForKey("id", &id)) { 133 success = false; 134 goto callback; 135 } 136 assert(!id.empty()); 137 138 size_t depth = std::count(id.begin(), id.end(), '/') - 1; 139 assert(depth < arraysize(LookupKey::kHierarchy)); 140 AddressField field = LookupKey::kHierarchy[depth]; 141 142 Rule* rule = new Rule; 143 if (field == COUNTRY) { 144 // All rules on the COUNTRY level inherit from the default rule. 145 rule->CopyFrom(Rule::GetDefault()); 146 } 147 rule->ParseJsonRule(*value); 148 assert(id == rule->GetId()); // Sanity check. 149 150 rule_storage_->push_back(rule); 151 if (depth > 0) { 152 sub_rules.push_back(rule); 153 } 154 155 // Add the ID of this Rule object to the rule index with natural string 156 // comparison for keys. 157 last_index_it = 158 rule_index_->insert(last_index_it, std::make_pair(id, rule)); 159 160 // Add the ID of this Rule object to the region-specific rule index with 161 // exact string comparison for keys. 162 last_region_it = 163 region_rules_->insert(last_region_it, std::make_pair(id, rule)); 164 165 ++rule_count; 166 } 167 168 /* 169 * Normally the address metadata server takes care of mapping from natural 170 * language names to metadata IDs (eg. "São Paulo" -> "SP") and from Latin 171 * script names to local script names (eg. "Tokushima" -> "徳島県"). 172 * 173 * As the PreloadSupplier doesn't contact the metadata server upon each 174 * Supply() request, it instead has an internal lookup table (rule_index_) 175 * that contains such mappings. 176 * 177 * This lookup table is populated by iterating over all sub rules and for 178 * each of them construct ID strings using human readable names (eg. "São 179 * Paulo") and using Latin script names (eg. "Tokushima"). 180 */ 181 for (std::vector<const Rule*>::const_iterator 182 it = sub_rules.begin(); it != sub_rules.end(); ++it) { 183 std::stack<const Rule*> hierarchy; 184 hierarchy.push(*it); 185 186 // Push pointers to all parent Rule objects onto the hierarchy stack. 187 for (std::string parent_id((*it)->GetId());;) { 188 // Strip the last part of parent_id. Break if COUNTRY level is reached. 189 std::string::size_type pos = parent_id.rfind('/'); 190 if (pos == sizeof "data/ZZ" - 1) { 191 break; 192 } 193 parent_id.resize(pos); 194 195 IndexMap::const_iterator* const hint = &hints[hierarchy.size() - 1]; 196 if (*hint == rule_index_->end() || (*hint)->first != parent_id) { 197 *hint = rule_index_->find(parent_id); 198 } 199 assert(*hint != rule_index_->end()); 200 hierarchy.push((*hint)->second); 201 } 202 203 std::string human_id((*it)->GetId().substr(0, sizeof "data/ZZ" - 1)); 204 std::string latin_id(human_id); 205 206 // Append the names from all Rule objects on the hierarchy stack. 207 for (; !hierarchy.empty(); hierarchy.pop()) { 208 const Rule* rule = hierarchy.top(); 209 210 human_id.push_back('/'); 211 if (!rule->GetName().empty()) { 212 human_id.append(rule->GetName()); 213 } else { 214 // If the "name" field is empty, the name is the last part of the ID. 215 const std::string& id = rule->GetId(); 216 std::string::size_type pos = id.rfind('/'); 217 assert(pos != std::string::npos); 218 human_id.append(id.substr(pos + 1)); 219 } 220 221 if (!rule->GetLatinName().empty()) { 222 latin_id.push_back('/'); 223 latin_id.append(rule->GetLatinName()); 224 } 225 } 226 227 // If the ID has a language tag, copy it. 228 { 229 const std::string& id = (*it)->GetId(); 230 std::string::size_type pos = id.rfind("--"); 231 if (pos != std::string::npos) { 232 human_id.append(id, pos, id.size() - pos); 233 } 234 } 235 236 last_index_it = 237 rule_index_->insert(last_index_it, std::make_pair(human_id, *it)); 238 239 // Add the Latin script ID, if a Latin script name could be found for 240 // every part of the ID. 241 if (std::count(human_id.begin(), human_id.end(), '/') == 242 std::count(latin_id.begin(), latin_id.end(), '/')) { 243 last_latin_it = 244 rule_index_->insert(last_latin_it, std::make_pair(latin_id, *it)); 245 } 246 } 247 248 callback: 249 loaded_(success, region_code_, rule_count); 250 delete this; 251 } 252 253 const std::string region_code_; 254 const PreloadSupplier::Callback& loaded_; 255 std::set<std::string>* const pending_; 256 IndexMap* const rule_index_; 257 std::vector<const Rule*>* const rule_storage_; 258 std::map<std::string, const Rule*>* const region_rules_; 259 const scoped_ptr<const Retriever::Callback> retrieved_; 260 261 DISALLOW_COPY_AND_ASSIGN(Helper); 262}; 263 264std::string KeyFromRegionCode(const std::string& region_code) { 265 AddressData address; 266 address.region_code = region_code; 267 LookupKey lookup_key; 268 lookup_key.FromAddress(address); 269 return lookup_key.ToKeyString(0); // Zero depth = COUNTRY level. 270} 271 272} // namespace 273 274PreloadSupplier::PreloadSupplier(const Source* source, Storage* storage) 275 : retriever_(new Retriever(source, storage)), 276 pending_(), 277 rule_index_(new IndexMap), 278 rule_storage_(), 279 region_rules_() {} 280 281PreloadSupplier::~PreloadSupplier() { 282 for (std::vector<const Rule*>::const_iterator 283 it = rule_storage_.begin(); it != rule_storage_.end(); ++it) { 284 delete *it; 285 } 286} 287 288void PreloadSupplier::Supply(const LookupKey& lookup_key, 289 const Supplier::Callback& supplied) { 290 Supplier::RuleHierarchy hierarchy; 291 bool success = GetRuleHierarchy(lookup_key, &hierarchy); 292 supplied(success, lookup_key, hierarchy); 293} 294 295const Rule* PreloadSupplier::GetRule(const LookupKey& lookup_key) const { 296 assert(IsLoaded(lookup_key.GetRegionCode())); 297 Supplier::RuleHierarchy hierarchy; 298 if (!GetRuleHierarchy(lookup_key, &hierarchy)) { 299 return NULL; 300 } 301 return hierarchy.rule[lookup_key.GetDepth()]; 302} 303 304void PreloadSupplier::LoadRules(const std::string& region_code, 305 const Callback& loaded) { 306 const std::string& key = KeyFromRegionCode(region_code); 307 308 if (IsLoadedKey(key)) { 309 loaded(true, region_code, 0); 310 return; 311 } 312 313 if (IsPendingKey(key)) { 314 return; 315 } 316 317 new Helper( 318 region_code, 319 key, 320 loaded, 321 *retriever_, 322 &pending_, 323 rule_index_.get(), 324 &rule_storage_, 325 ®ion_rules_[region_code]); 326} 327 328const std::map<std::string, const Rule*>& PreloadSupplier::GetRulesForRegion( 329 const std::string& region_code) const { 330 assert(IsLoaded(region_code)); 331 return region_rules_.find(region_code)->second; 332} 333 334bool PreloadSupplier::IsLoaded(const std::string& region_code) const { 335 return IsLoadedKey(KeyFromRegionCode(region_code)); 336} 337 338bool PreloadSupplier::IsPending(const std::string& region_code) const { 339 return IsPendingKey(KeyFromRegionCode(region_code)); 340} 341 342bool PreloadSupplier::GetRuleHierarchy(const LookupKey& lookup_key, 343 RuleHierarchy* hierarchy) const { 344 assert(hierarchy != NULL); 345 346 if (RegionDataConstants::IsSupported(lookup_key.GetRegionCode())) { 347 size_t max_depth = std::min( 348 lookup_key.GetDepth(), 349 RegionDataConstants::GetMaxLookupKeyDepth(lookup_key.GetRegionCode())); 350 351 for (size_t depth = 0; depth <= max_depth; ++depth) { 352 const std::string& key = lookup_key.ToKeyString(depth); 353 IndexMap::const_iterator it = rule_index_->find(key); 354 if (it == rule_index_->end()) { 355 return depth > 0; // No data on COUNTRY level is failure. 356 } 357 hierarchy->rule[depth] = it->second; 358 } 359 } 360 361 return true; 362} 363 364bool PreloadSupplier::IsLoadedKey(const std::string& key) const { 365 return rule_index_->find(key) != rule_index_->end(); 366} 367 368bool PreloadSupplier::IsPendingKey(const std::string& key) const { 369 return pending_.find(key) != pending_.end(); 370} 371 372} // namespace addressinput 373} // namespace i18n 374