ImapStore.java revision 35b0e95ca795e17b6dc8dd98c7ab847d65d9aa0c
1/* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.email.mail.store; 18 19import android.content.Context; 20import android.os.Build; 21import android.os.Bundle; 22import android.telephony.TelephonyManager; 23import android.text.TextUtils; 24import android.util.Base64; 25import android.util.Log; 26 27import com.android.email.LegacyConversions; 28import com.android.email.Preferences; 29import com.android.email.VendorPolicyLoader; 30import com.android.email.mail.Store; 31import com.android.email.mail.Transport; 32import com.android.email.mail.store.imap.ImapConstants; 33import com.android.email.mail.store.imap.ImapResponse; 34import com.android.email.mail.store.imap.ImapString; 35import com.android.email.mail.transport.MailTransport; 36import com.android.emailcommon.Logging; 37import com.android.emailcommon.internet.MimeMessage; 38import com.android.emailcommon.mail.AuthenticationFailedException; 39import com.android.emailcommon.mail.Flag; 40import com.android.emailcommon.mail.Folder; 41import com.android.emailcommon.mail.Message; 42import com.android.emailcommon.mail.MessagingException; 43import com.android.emailcommon.provider.Account; 44import com.android.emailcommon.provider.HostAuth; 45import com.android.emailcommon.provider.Mailbox; 46import com.android.emailcommon.service.EmailServiceProxy; 47import com.android.emailcommon.utility.Utility; 48import com.beetstra.jutf7.CharsetProvider; 49import com.google.common.annotations.VisibleForTesting; 50 51import java.io.IOException; 52import java.io.InputStream; 53import java.nio.ByteBuffer; 54import java.nio.charset.Charset; 55import java.security.MessageDigest; 56import java.security.NoSuchAlgorithmException; 57import java.util.Collection; 58import java.util.HashMap; 59import java.util.List; 60import java.util.Set; 61import java.util.concurrent.ConcurrentLinkedQueue; 62import java.util.regex.Pattern; 63 64 65/** 66 * <pre> 67 * TODO Need to start keeping track of UIDVALIDITY 68 * TODO Need a default response handler for things like folder updates 69 * TODO In fetch(), if we need a ImapMessage and were given 70 * something else we can try to do a pre-fetch first. 71 * TODO Collect ALERT messages and show them to users. 72 * 73 * ftp://ftp.isi.edu/in-notes/rfc2683.txt When a client asks for 74 * certain information in a FETCH command, the server may return the requested 75 * information in any order, not necessarily in the order that it was requested. 76 * Further, the server may return the information in separate FETCH responses 77 * and may also return information that was not explicitly requested (to reflect 78 * to the client changes in the state of the subject message). 79 * </pre> 80 */ 81public class ImapStore extends Store { 82 /** Charset used for converting folder names to and from UTF-7 as defined by RFC 3501. */ 83 private static final Charset MODIFIED_UTF_7_CHARSET = 84 new CharsetProvider().charsetForName("X-RFC-3501"); 85 86 @VisibleForTesting static String sImapId = null; 87 @VisibleForTesting String mPathPrefix; 88 @VisibleForTesting String mPathSeparator; 89 90 private final ConcurrentLinkedQueue<ImapConnection> mConnectionPool = 91 new ConcurrentLinkedQueue<ImapConnection>(); 92 93 /** 94 * Cache of ImapFolder objects. ImapFolders are attached to a given folder on the server 95 * and as long as their associated connection remains open they are reusable between 96 * requests. This cache lets us make sure we always reuse, if possible, for a given 97 * folder name. 98 */ 99 private final HashMap<String, ImapFolder> mFolderCache = new HashMap<String, ImapFolder>(); 100 101 /** 102 * Static named constructor. 103 */ 104 public static Store newInstance(Account account, Context context) throws MessagingException { 105 return new ImapStore(context, account); 106 } 107 108 /** 109 * Creates a new store for the given account. Always use 110 * {@link #newInstance(Account, Context, PersistentDataCallbacks)} to create an IMAP store. 111 */ 112 private ImapStore(Context context, Account account) throws MessagingException { 113 mContext = context; 114 mAccount = account; 115 116 HostAuth recvAuth = account.getOrCreateHostAuthRecv(context); 117 if (recvAuth == null || !STORE_SCHEME_IMAP.equalsIgnoreCase(recvAuth.mProtocol)) { 118 throw new MessagingException("Unsupported protocol"); 119 } 120 // defaults, which can be changed by security modifiers 121 int connectionSecurity = Transport.CONNECTION_SECURITY_NONE; 122 int defaultPort = 143; 123 124 // check for security flags and apply changes 125 if ((recvAuth.mFlags & HostAuth.FLAG_SSL) != 0) { 126 connectionSecurity = Transport.CONNECTION_SECURITY_SSL; 127 defaultPort = 993; 128 } else if ((recvAuth.mFlags & HostAuth.FLAG_TLS) != 0) { 129 connectionSecurity = Transport.CONNECTION_SECURITY_TLS; 130 } 131 boolean trustCertificates = ((recvAuth.mFlags & HostAuth.FLAG_TRUST_ALL) != 0); 132 int port = defaultPort; 133 if (recvAuth.mPort != HostAuth.PORT_UNKNOWN) { 134 port = recvAuth.mPort; 135 } 136 mTransport = new MailTransport("IMAP"); 137 mTransport.setHost(recvAuth.mAddress); 138 mTransport.setPort(port); 139 mTransport.setSecurity(connectionSecurity, trustCertificates); 140 141 String[] userInfo = recvAuth.getLogin(); 142 if (userInfo != null) { 143 mUsername = userInfo[0]; 144 mPassword = userInfo[1]; 145 } else { 146 mUsername = null; 147 mPassword = null; 148 } 149 mPathPrefix = recvAuth.mDomain; 150 } 151 152 @VisibleForTesting 153 Collection<ImapConnection> getConnectionPoolForTest() { 154 return mConnectionPool; 155 } 156 157 /** 158 * For testing only. Injects a different root transport (it will be copied using 159 * newInstanceWithConfiguration() each time IMAP sets up a new channel). The transport 160 * should already be set up and ready to use. Do not use for real code. 161 * @param testTransport The Transport to inject and use for all future communication. 162 */ 163 @VisibleForTesting 164 void setTransportForTest(Transport testTransport) { 165 mTransport = testTransport; 166 } 167 168 /** 169 * Return, or create and return, an string suitable for use in an IMAP ID message. 170 * This is constructed similarly to the way the browser sets up its user-agent strings. 171 * See RFC 2971 for more details. The output of this command will be a series of key-value 172 * pairs delimited by spaces (there is no point in returning a structured result because 173 * this will be sent as-is to the IMAP server). No tokens, parenthesis or "ID" are included, 174 * because some connections may append additional values. 175 * 176 * The following IMAP ID keys may be included: 177 * name Android package name of the program 178 * os "android" 179 * os-version "version; model; build-id" 180 * vendor Vendor of the client/server 181 * x-android-device-model Model (only revealed if release build) 182 * x-android-net-operator Mobile network operator (if known) 183 * AGUID A device+account UID 184 * 185 * In addition, a vendor policy .apk can append key/value pairs. 186 * 187 * @param userName the username of the account 188 * @param host the host (server) of the account 189 * @param capabilities a list of the capabilities from the server 190 * @return a String for use in an IMAP ID message. 191 */ 192 @VisibleForTesting 193 static String getImapId(Context context, String userName, String host, String capabilities) { 194 // The first section is global to all IMAP connections, and generates the fixed 195 // values in any IMAP ID message 196 synchronized (ImapStore.class) { 197 if (sImapId == null) { 198 TelephonyManager tm = 199 (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); 200 String networkOperator = tm.getNetworkOperatorName(); 201 if (networkOperator == null) networkOperator = ""; 202 203 sImapId = makeCommonImapId(context.getPackageName(), Build.VERSION.RELEASE, 204 Build.VERSION.CODENAME, Build.MODEL, Build.ID, Build.MANUFACTURER, 205 networkOperator); 206 } 207 } 208 209 // This section is per Store, and adds in a dynamic elements like UID's. 210 // We don't cache the result of this work, because the caller does anyway. 211 StringBuilder id = new StringBuilder(sImapId); 212 213 // Optionally add any vendor-supplied id keys 214 String vendorId = VendorPolicyLoader.getInstance(context).getImapIdValues(userName, host, 215 capabilities); 216 if (vendorId != null) { 217 id.append(' '); 218 id.append(vendorId); 219 } 220 221 // Generate a UID that mixes a "stable" device UID with the email address 222 try { 223 String devUID = Preferences.getPreferences(context).getDeviceUID(); 224 MessageDigest messageDigest; 225 messageDigest = MessageDigest.getInstance("SHA-1"); 226 messageDigest.update(userName.getBytes()); 227 messageDigest.update(devUID.getBytes()); 228 byte[] uid = messageDigest.digest(); 229 String hexUid = Base64.encodeToString(uid, Base64.NO_WRAP); 230 id.append(" \"AGUID\" \""); 231 id.append(hexUid); 232 id.append('\"'); 233 } catch (NoSuchAlgorithmException e) { 234 Log.d(Logging.LOG_TAG, "couldn't obtain SHA-1 hash for device UID"); 235 } 236 return id.toString(); 237 } 238 239 /** 240 * Helper function that actually builds the static part of the IMAP ID string. This is 241 * separated from getImapId for testability. There is no escaping or encoding in IMAP ID so 242 * any rogue chars must be filtered here. 243 * 244 * @param packageName context.getPackageName() 245 * @param version Build.VERSION.RELEASE 246 * @param codeName Build.VERSION.CODENAME 247 * @param model Build.MODEL 248 * @param id Build.ID 249 * @param vendor Build.MANUFACTURER 250 * @param networkOperator TelephonyManager.getNetworkOperatorName() 251 * @return the static (never changes) portion of the IMAP ID 252 */ 253 @VisibleForTesting 254 static String makeCommonImapId(String packageName, String version, 255 String codeName, String model, String id, String vendor, String networkOperator) { 256 257 // Before building up IMAP ID string, pre-filter the input strings for "legal" chars 258 // This is using a fairly arbitrary char set intended to pass through most reasonable 259 // version, model, and vendor strings: a-z A-Z 0-9 - _ + = ; : . , / <space> 260 // The most important thing is *not* to pass parens, quotes, or CRLF, which would break 261 // the format of the IMAP ID list. 262 Pattern p = Pattern.compile("[^a-zA-Z0-9-_\\+=;:\\.,/ ]"); 263 packageName = p.matcher(packageName).replaceAll(""); 264 version = p.matcher(version).replaceAll(""); 265 codeName = p.matcher(codeName).replaceAll(""); 266 model = p.matcher(model).replaceAll(""); 267 id = p.matcher(id).replaceAll(""); 268 vendor = p.matcher(vendor).replaceAll(""); 269 networkOperator = p.matcher(networkOperator).replaceAll(""); 270 271 // "name" "com.android.email" 272 StringBuffer sb = new StringBuffer("\"name\" \""); 273 sb.append(packageName); 274 sb.append("\""); 275 276 // "os" "android" 277 sb.append(" \"os\" \"android\""); 278 279 // "os-version" "version; build-id" 280 sb.append(" \"os-version\" \""); 281 if (version.length() > 0) { 282 sb.append(version); 283 } else { 284 // default to "1.0" 285 sb.append("1.0"); 286 } 287 // add the build ID or build # 288 if (id.length() > 0) { 289 sb.append("; "); 290 sb.append(id); 291 } 292 sb.append("\""); 293 294 // "vendor" "the vendor" 295 if (vendor.length() > 0) { 296 sb.append(" \"vendor\" \""); 297 sb.append(vendor); 298 sb.append("\""); 299 } 300 301 // "x-android-device-model" the device model (on release builds only) 302 if ("REL".equals(codeName)) { 303 if (model.length() > 0) { 304 sb.append(" \"x-android-device-model\" \""); 305 sb.append(model); 306 sb.append("\""); 307 } 308 } 309 310 // "x-android-mobile-net-operator" "name of network operator" 311 if (networkOperator.length() > 0) { 312 sb.append(" \"x-android-mobile-net-operator\" \""); 313 sb.append(networkOperator); 314 sb.append("\""); 315 } 316 317 return sb.toString(); 318 } 319 320 321 @Override 322 public Folder getFolder(String name) { 323 ImapFolder folder; 324 synchronized (mFolderCache) { 325 folder = mFolderCache.get(name); 326 if (folder == null) { 327 folder = new ImapFolder(this, name); 328 mFolderCache.put(name, folder); 329 } 330 } 331 return folder; 332 } 333 334 /** 335 * Creates a mailbox hierarchy out of the flat data provided by the server. 336 */ 337 @VisibleForTesting 338 static void createHierarchy(HashMap<String, ImapFolder> mailboxes) { 339 Set<String> pathnames = mailboxes.keySet(); 340 for (String path : pathnames) { 341 final ImapFolder folder = mailboxes.get(path); 342 final Mailbox mailbox = folder.mMailbox; 343 int delimiterIdx = mailbox.mServerId.lastIndexOf(mailbox.mDelimiter); 344 long parentKey = -1L; 345 if (delimiterIdx != -1) { 346 String parentPath = path.substring(0, delimiterIdx); 347 final ImapFolder parentFolder = mailboxes.get(parentPath); 348 final Mailbox parentMailbox = (parentFolder == null) ? null : parentFolder.mMailbox; 349 if (parentMailbox != null) { 350 parentKey = parentMailbox.mId; 351 parentMailbox.mFlags 352 |= (Mailbox.FLAG_HAS_CHILDREN | Mailbox.FLAG_CHILDREN_VISIBLE); 353 } 354 } 355 mailbox.mParentKey = parentKey; 356 } 357 } 358 359 /** 360 * Creates a {@link Folder} and associated {@link Mailbox}. If the folder does not already 361 * exist in the local database, a new row will immediately be created in the mailbox table. 362 * Otherwise, the existing row will be used. Any changes to existing rows, will not be stored 363 * to the database immediately. 364 * @param accountId The ID of the account the mailbox is to be associated with 365 * @param mailboxPath The path of the mailbox to add 366 * @param delimiter A path delimiter. May be {@code null} if there is no delimiter. 367 * @param selectable If {@code true}, the mailbox can be selected and used to store messages. 368 */ 369 private ImapFolder addMailbox(Context context, long accountId, String mailboxPath, 370 char delimiter, boolean selectable) { 371 ImapFolder folder = (ImapFolder) getFolder(mailboxPath); 372 Mailbox mailbox = getMailboxForPath(context, accountId, mailboxPath); 373 if (mailbox.isSaved()) { 374 // existing mailbox 375 // mailbox retrieved from database; save hash _before_ updating fields 376 folder.mHash = mailbox.getHashes(); 377 } 378 updateMailbox(mailbox, accountId, mailboxPath, delimiter, selectable, 379 LegacyConversions.inferMailboxTypeFromName(context, mailboxPath)); 380 if (folder.mHash == null) { 381 // new mailbox 382 // save hash after updating. allows tracking changes if the mailbox is saved 383 // outside of #saveMailboxList() 384 folder.mHash = mailbox.getHashes(); 385 // We must save this here to make sure we have a valid ID for later 386 mailbox.save(mContext); 387 } 388 folder.mMailbox = mailbox; 389 return folder; 390 } 391 392 /** 393 * Persists the folders in the given list. 394 */ 395 private static void saveMailboxList(Context context, HashMap<String, ImapFolder> folderMap) { 396 for (ImapFolder imapFolder : folderMap.values()) { 397 imapFolder.save(context); 398 } 399 } 400 401 @Override 402 public Folder[] updateFolders() throws MessagingException { 403 ImapConnection connection = getConnection(); 404 try { 405 HashMap<String, ImapFolder> mailboxes = new HashMap<String, ImapFolder>(); 406 // Establish a connection to the IMAP server; if necessary 407 // This ensures a valid prefix if the prefix is automatically set by the server 408 connection.executeSimpleCommand(ImapConstants.NOOP); 409 String imapCommand = ImapConstants.LIST + " \"\" \"*\""; 410 if (mPathPrefix != null) { 411 imapCommand = ImapConstants.LIST + " \"\" \"" + mPathPrefix + "*\""; 412 } 413 List<ImapResponse> responses = connection.executeSimpleCommand(imapCommand); 414 for (ImapResponse response : responses) { 415 // S: * LIST (\Noselect) "/" ~/Mail/foo 416 if (response.isDataResponse(0, ImapConstants.LIST)) { 417 // Get folder name. 418 ImapString encodedFolder = response.getStringOrEmpty(3); 419 if (encodedFolder.isEmpty()) continue; 420 421 String folderName = decodeFolderName(encodedFolder.getString(), mPathPrefix); 422 if (ImapConstants.INBOX.equalsIgnoreCase(folderName)) continue; 423 424 // Parse attributes. 425 boolean selectable = 426 !response.getListOrEmpty(1).contains(ImapConstants.FLAG_NO_SELECT); 427 String delimiter = response.getStringOrEmpty(2).getString(); 428 char delimiterChar = '\0'; 429 if (!TextUtils.isEmpty(delimiter)) { 430 delimiterChar = delimiter.charAt(0); 431 } 432 ImapFolder folder = 433 addMailbox(mContext, mAccount.mId, folderName, delimiterChar, selectable); 434 mailboxes.put(folderName, folder); 435 } 436 } 437 Folder newFolder = 438 addMailbox(mContext, mAccount.mId, ImapConstants.INBOX, '\0', true /*selectable*/); 439 mailboxes.put(ImapConstants.INBOX, (ImapFolder)newFolder); 440 createHierarchy(mailboxes); 441 saveMailboxList(mContext, mailboxes); 442 return mailboxes.values().toArray(new Folder[] {}); 443 } catch (IOException ioe) { 444 connection.close(); 445 throw new MessagingException("Unable to get folder list.", ioe); 446 } catch (AuthenticationFailedException afe) { 447 // We do NOT want this connection pooled, or we will continue to send NOOP and SELECT 448 // commands to the server 449 connection.destroyResponses(); 450 connection = null; 451 throw afe; 452 } finally { 453 if (connection != null) { 454 poolConnection(connection); 455 } 456 } 457 } 458 459 @Override 460 public Bundle checkSettings() throws MessagingException { 461 int result = MessagingException.NO_ERROR; 462 Bundle bundle = new Bundle(); 463 ImapConnection connection = new ImapConnection(this, mUsername, mPassword); 464 try { 465 connection.open(); 466 connection.close(); 467 } catch (IOException ioe) { 468 bundle.putString(EmailServiceProxy.VALIDATE_BUNDLE_ERROR_MESSAGE, ioe.getMessage()); 469 result = MessagingException.IOERROR; 470 } finally { 471 connection.destroyResponses(); 472 } 473 bundle.putInt(EmailServiceProxy.VALIDATE_BUNDLE_RESULT_CODE, result); 474 return bundle; 475 } 476 477 /** 478 * Returns whether or not the prefix has been set by the user. This can be determined by 479 * the fact that the prefix is set, but, the path separator is not set. 480 */ 481 boolean isUserPrefixSet() { 482 return TextUtils.isEmpty(mPathSeparator) && !TextUtils.isEmpty(mPathPrefix); 483 } 484 485 /** Sets the path separator */ 486 void setPathSeparator(String pathSeparator) { 487 mPathSeparator = pathSeparator; 488 } 489 490 /** Sets the prefix */ 491 void setPathPrefix(String pathPrefix) { 492 mPathPrefix = pathPrefix; 493 } 494 495 /** Gets the context for this store */ 496 Context getContext() { 497 return mContext; 498 } 499 500 /** Returns a clone of the transport associated with this store. */ 501 Transport cloneTransport() { 502 return mTransport.clone(); 503 } 504 505 /** 506 * Fixes the path prefix, if necessary. The path prefix must always end with the 507 * path separator. 508 */ 509 void ensurePrefixIsValid() { 510 // Make sure the path prefix ends with the path separator 511 if (!TextUtils.isEmpty(mPathPrefix) && !TextUtils.isEmpty(mPathSeparator)) { 512 if (!mPathPrefix.endsWith(mPathSeparator)) { 513 mPathPrefix = mPathPrefix + mPathSeparator; 514 } 515 } 516 } 517 518 /** 519 * Gets a connection if one is available from the pool, or creates a new one if not. 520 */ 521 ImapConnection getConnection() { 522 ImapConnection connection = null; 523 while ((connection = mConnectionPool.poll()) != null) { 524 try { 525 connection.setStore(this, mUsername, mPassword); 526 connection.executeSimpleCommand(ImapConstants.NOOP); 527 break; 528 } catch (MessagingException e) { 529 // Fall through 530 } catch (IOException e) { 531 // Fall through 532 } 533 connection.close(); 534 connection = null; 535 } 536 if (connection == null) { 537 connection = new ImapConnection(this, mUsername, mPassword); 538 } 539 return connection; 540 } 541 542 /** 543 * Save a {@link ImapConnection} in the pool for reuse. Any responses associated with the 544 * connection are destroyed before adding the connection to the pool. 545 */ 546 void poolConnection(ImapConnection connection) { 547 if (connection != null) { 548 connection.destroyResponses(); 549 mConnectionPool.add(connection); 550 } 551 } 552 553 /** 554 * Prepends the folder name with the given prefix and UTF-7 encodes it. 555 */ 556 static String encodeFolderName(String name, String prefix) { 557 // do NOT add the prefix to the special name "INBOX" 558 if (ImapConstants.INBOX.equalsIgnoreCase(name)) return name; 559 560 // Prepend prefix 561 if (prefix != null) { 562 name = prefix + name; 563 } 564 565 // TODO bypass the conversion if name doesn't have special char. 566 ByteBuffer bb = MODIFIED_UTF_7_CHARSET.encode(name); 567 byte[] b = new byte[bb.limit()]; 568 bb.get(b); 569 570 return Utility.fromAscii(b); 571 } 572 573 /** 574 * UTF-7 decodes the folder name and removes the given path prefix. 575 */ 576 static String decodeFolderName(String name, String prefix) { 577 // TODO bypass the conversion if name doesn't have special char. 578 String folder; 579 folder = MODIFIED_UTF_7_CHARSET.decode(ByteBuffer.wrap(Utility.toAscii(name))).toString(); 580 if ((prefix != null) && folder.startsWith(prefix)) { 581 folder = folder.substring(prefix.length()); 582 } 583 return folder; 584 } 585 586 /** 587 * Returns UIDs of Messages joined with "," as the separator. 588 */ 589 static String joinMessageUids(Message[] messages) { 590 StringBuilder sb = new StringBuilder(); 591 boolean notFirst = false; 592 for (Message m : messages) { 593 if (notFirst) { 594 sb.append(','); 595 } 596 sb.append(m.getUid()); 597 notFirst = true; 598 } 599 return sb.toString(); 600 } 601 602 static class ImapMessage extends MimeMessage { 603 ImapMessage(String uid, ImapFolder folder) { 604 this.mUid = uid; 605 this.mFolder = folder; 606 } 607 608 public void setSize(int size) { 609 this.mSize = size; 610 } 611 612 @Override 613 public void parse(InputStream in) throws IOException, MessagingException { 614 super.parse(in); 615 } 616 617 public void setFlagInternal(Flag flag, boolean set) throws MessagingException { 618 super.setFlag(flag, set); 619 } 620 621 @Override 622 public void setFlag(Flag flag, boolean set) throws MessagingException { 623 super.setFlag(flag, set); 624 mFolder.setFlags(new Message[] { this }, new Flag[] { flag }, set); 625 } 626 } 627 628 static class ImapException extends MessagingException { 629 private static final long serialVersionUID = 1L; 630 631 String mAlertText; 632 633 public ImapException(String message, String alertText, Throwable throwable) { 634 super(message, throwable); 635 this.mAlertText = alertText; 636 } 637 638 public ImapException(String message, String alertText) { 639 super(message); 640 this.mAlertText = alertText; 641 } 642 643 public String getAlertText() { 644 return mAlertText; 645 } 646 647 public void setAlertText(String alertText) { 648 mAlertText = alertText; 649 } 650 } 651} 652