1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.android.email.mail.store; 18 19import android.content.Context; 20import android.text.TextUtils; 21import android.util.Base64DataException; 22 23import com.android.email.DebugUtils; 24import com.android.email.mail.store.ImapStore.ImapException; 25import com.android.email.mail.store.ImapStore.ImapMessage; 26import com.android.email.mail.store.imap.ImapConstants; 27import com.android.email.mail.store.imap.ImapElement; 28import com.android.email.mail.store.imap.ImapList; 29import com.android.email.mail.store.imap.ImapResponse; 30import com.android.email.mail.store.imap.ImapString; 31import com.android.email.mail.store.imap.ImapUtility; 32import com.android.email.service.ImapService; 33import com.android.emailcommon.Logging; 34import com.android.emailcommon.internet.BinaryTempFileBody; 35import com.android.emailcommon.internet.MimeBodyPart; 36import com.android.emailcommon.internet.MimeHeader; 37import com.android.emailcommon.internet.MimeMultipart; 38import com.android.emailcommon.internet.MimeUtility; 39import com.android.emailcommon.mail.AuthenticationFailedException; 40import com.android.emailcommon.mail.Body; 41import com.android.emailcommon.mail.FetchProfile; 42import com.android.emailcommon.mail.Flag; 43import com.android.emailcommon.mail.Folder; 44import com.android.emailcommon.mail.Message; 45import com.android.emailcommon.mail.MessagingException; 46import com.android.emailcommon.mail.Part; 47import com.android.emailcommon.provider.Mailbox; 48import com.android.emailcommon.service.SearchParams; 49import com.android.emailcommon.utility.CountingOutputStream; 50import com.android.emailcommon.utility.EOLConvertingOutputStream; 51import com.android.emailcommon.utility.Utility; 52import com.android.mail.utils.LogUtils; 53import com.google.common.annotations.VisibleForTesting; 54 55import org.apache.commons.io.IOUtils; 56 57import java.io.File; 58import java.io.FileInputStream; 59import java.io.FileOutputStream; 60import java.io.IOException; 61import java.io.InputStream; 62import java.io.OutputStream; 63import java.text.SimpleDateFormat; 64import java.util.ArrayList; 65import java.util.Arrays; 66import java.util.Date; 67import java.util.HashMap; 68import java.util.LinkedHashSet; 69import java.util.List; 70import java.util.Locale; 71import java.util.TimeZone; 72 73class ImapFolder extends Folder { 74 private final static Flag[] PERMANENT_FLAGS = 75 { Flag.DELETED, Flag.SEEN, Flag.FLAGGED, Flag.ANSWERED }; 76 private static final int COPY_BUFFER_SIZE = 16*1024; 77 78 private final ImapStore mStore; 79 private final String mName; 80 private int mMessageCount = -1; 81 private ImapConnection mConnection; 82 private OpenMode mMode; 83 private boolean mExists; 84 /** The local mailbox associated with this remote folder */ 85 Mailbox mMailbox; 86 /** A set of hashes that can be used to track dirtiness */ 87 Object mHash[]; 88 89 /*package*/ ImapFolder(ImapStore store, String name) { 90 mStore = store; 91 mName = name; 92 } 93 94 private void destroyResponses() { 95 if (mConnection != null) { 96 mConnection.destroyResponses(); 97 } 98 } 99 100 @Override 101 public void open(OpenMode mode) 102 throws MessagingException { 103 try { 104 if (isOpen()) { 105 if (mMode == mode) { 106 // Make sure the connection is valid. 107 // If it's not we'll close it down and continue on to get a new one. 108 try { 109 mConnection.executeSimpleCommand(ImapConstants.NOOP); 110 return; 111 112 } catch (IOException ioe) { 113 ioExceptionHandler(mConnection, ioe); 114 } finally { 115 destroyResponses(); 116 } 117 } else { 118 // Return the connection to the pool, if exists. 119 close(false); 120 } 121 } 122 synchronized (this) { 123 mConnection = mStore.getConnection(); 124 } 125 // * FLAGS (\Answered \Flagged \Deleted \Seen \Draft NonJunk 126 // $MDNSent) 127 // * OK [PERMANENTFLAGS (\Answered \Flagged \Deleted \Seen \Draft 128 // NonJunk $MDNSent \*)] Flags permitted. 129 // * 23 EXISTS 130 // * 0 RECENT 131 // * OK [UIDVALIDITY 1125022061] UIDs valid 132 // * OK [UIDNEXT 57576] Predicted next UID 133 // 2 OK [READ-WRITE] Select completed. 134 try { 135 doSelect(); 136 } catch (IOException ioe) { 137 throw ioExceptionHandler(mConnection, ioe); 138 } finally { 139 destroyResponses(); 140 } 141 } catch (AuthenticationFailedException e) { 142 // Don't cache this connection, so we're forced to try connecting/login again 143 mConnection = null; 144 close(false); 145 throw e; 146 } catch (MessagingException e) { 147 mExists = false; 148 close(false); 149 throw e; 150 } 151 } 152 153 @Override 154 @VisibleForTesting 155 public boolean isOpen() { 156 return mExists && mConnection != null; 157 } 158 159 @Override 160 public OpenMode getMode() { 161 return mMode; 162 } 163 164 @Override 165 public void close(boolean expunge) { 166 // TODO implement expunge 167 mMessageCount = -1; 168 synchronized (this) { 169 mStore.poolConnection(mConnection); 170 mConnection = null; 171 } 172 } 173 174 @Override 175 public String getName() { 176 return mName; 177 } 178 179 @Override 180 public boolean exists() throws MessagingException { 181 if (mExists) { 182 return true; 183 } 184 /* 185 * This method needs to operate in the unselected mode as well as the selected mode 186 * so we must get the connection ourselves if it's not there. We are specifically 187 * not calling checkOpen() since we don't care if the folder is open. 188 */ 189 ImapConnection connection = null; 190 synchronized(this) { 191 if (mConnection == null) { 192 connection = mStore.getConnection(); 193 } else { 194 connection = mConnection; 195 } 196 } 197 try { 198 connection.executeSimpleCommand(String.format(Locale.US, 199 ImapConstants.STATUS + " \"%s\" (" + ImapConstants.UIDVALIDITY + ")", 200 ImapStore.encodeFolderName(mName, mStore.mPathPrefix))); 201 mExists = true; 202 return true; 203 204 } catch (MessagingException me) { 205 // Treat IOERROR messaging exception as IOException 206 if (me.getExceptionType() == MessagingException.IOERROR) { 207 throw me; 208 } 209 return false; 210 211 } catch (IOException ioe) { 212 throw ioExceptionHandler(connection, ioe); 213 214 } finally { 215 connection.destroyResponses(); 216 if (mConnection == null) { 217 mStore.poolConnection(connection); 218 } 219 } 220 } 221 222 // IMAP supports folder creation 223 @Override 224 public boolean canCreate(FolderType type) { 225 return true; 226 } 227 228 @Override 229 public boolean create(FolderType type) throws MessagingException { 230 /* 231 * This method needs to operate in the unselected mode as well as the selected mode 232 * so we must get the connection ourselves if it's not there. We are specifically 233 * not calling checkOpen() since we don't care if the folder is open. 234 */ 235 ImapConnection connection = null; 236 synchronized(this) { 237 if (mConnection == null) { 238 connection = mStore.getConnection(); 239 } else { 240 connection = mConnection; 241 } 242 } 243 try { 244 connection.executeSimpleCommand(String.format(Locale.US, 245 ImapConstants.CREATE + " \"%s\"", 246 ImapStore.encodeFolderName(mName, mStore.mPathPrefix))); 247 return true; 248 249 } catch (MessagingException me) { 250 return false; 251 252 } catch (IOException ioe) { 253 throw ioExceptionHandler(connection, ioe); 254 255 } finally { 256 connection.destroyResponses(); 257 if (mConnection == null) { 258 mStore.poolConnection(connection); 259 } 260 } 261 } 262 263 @Override 264 public void copyMessages(Message[] messages, Folder folder, 265 MessageUpdateCallbacks callbacks) throws MessagingException { 266 checkOpen(); 267 try { 268 List<ImapResponse> responseList = mConnection.executeSimpleCommand( 269 String.format(Locale.US, ImapConstants.UID_COPY + " %s \"%s\"", 270 ImapStore.joinMessageUids(messages), 271 ImapStore.encodeFolderName(folder.getName(), mStore.mPathPrefix))); 272 // Build a message map for faster UID matching 273 HashMap<String, Message> messageMap = new HashMap<String, Message>(); 274 boolean handledUidPlus = false; 275 for (Message m : messages) { 276 messageMap.put(m.getUid(), m); 277 } 278 // Process response to get the new UIDs 279 for (ImapResponse response : responseList) { 280 // All "BAD" responses are bad. Only "NO", tagged responses are bad. 281 if (response.isBad() || (response.isNo() && response.isTagged())) { 282 String responseText = response.getStatusResponseTextOrEmpty().getString(); 283 throw new MessagingException(responseText); 284 } 285 // Skip untagged responses; they're just status 286 if (!response.isTagged()) { 287 continue; 288 } 289 // No callback provided to report of UID changes; nothing more to do here 290 // NOTE: We check this here to catch any server errors 291 if (callbacks == null) { 292 continue; 293 } 294 ImapList copyResponse = response.getListOrEmpty(1); 295 String responseCode = copyResponse.getStringOrEmpty(0).getString(); 296 if (ImapConstants.COPYUID.equals(responseCode)) { 297 handledUidPlus = true; 298 String origIdSet = copyResponse.getStringOrEmpty(2).getString(); 299 String newIdSet = copyResponse.getStringOrEmpty(3).getString(); 300 String[] origIdArray = ImapUtility.getImapSequenceValues(origIdSet); 301 String[] newIdArray = ImapUtility.getImapSequenceValues(newIdSet); 302 // There has to be a 1:1 mapping between old and new IDs 303 if (origIdArray.length != newIdArray.length) { 304 throw new MessagingException("Set length mis-match; orig IDs \"" + 305 origIdSet + "\" new IDs \"" + newIdSet + "\""); 306 } 307 for (int i = 0; i < origIdArray.length; i++) { 308 final String id = origIdArray[i]; 309 final Message m = messageMap.get(id); 310 if (m != null) { 311 callbacks.onMessageUidChange(m, newIdArray[i]); 312 } 313 } 314 } 315 } 316 // If the server doesn't support UIDPLUS, try a different way to get the new UID(s) 317 if (callbacks != null && !handledUidPlus) { 318 final ImapFolder newFolder = (ImapFolder)folder; 319 try { 320 // Temporarily select the destination folder 321 newFolder.open(OpenMode.READ_WRITE); 322 // Do the search(es) ... 323 for (Message m : messages) { 324 final String searchString = 325 "HEADER Message-Id \"" + m.getMessageId() + "\""; 326 final String[] newIdArray = newFolder.searchForUids(searchString); 327 if (newIdArray.length == 1) { 328 callbacks.onMessageUidChange(m, newIdArray[0]); 329 } 330 } 331 } catch (MessagingException e) { 332 // Log, but, don't abort; failures here don't need to be propagated 333 LogUtils.d(Logging.LOG_TAG, "Failed to find message", e); 334 } finally { 335 newFolder.close(false); 336 } 337 // Re-select the original folder 338 doSelect(); 339 } 340 } catch (IOException ioe) { 341 throw ioExceptionHandler(mConnection, ioe); 342 } finally { 343 destroyResponses(); 344 } 345 } 346 347 @Override 348 public int getMessageCount() { 349 return mMessageCount; 350 } 351 352 @Override 353 public int getUnreadMessageCount() throws MessagingException { 354 checkOpen(); 355 try { 356 int unreadMessageCount = 0; 357 final List<ImapResponse> responses = mConnection.executeSimpleCommand( 358 String.format(Locale.US, 359 ImapConstants.STATUS + " \"%s\" (" + ImapConstants.UNSEEN + ")", 360 ImapStore.encodeFolderName(mName, mStore.mPathPrefix))); 361 // S: * STATUS mboxname (MESSAGES 231 UIDNEXT 44292) 362 for (ImapResponse response : responses) { 363 if (response.isDataResponse(0, ImapConstants.STATUS)) { 364 unreadMessageCount = response.getListOrEmpty(2) 365 .getKeyedStringOrEmpty(ImapConstants.UNSEEN).getNumberOrZero(); 366 } 367 } 368 return unreadMessageCount; 369 } catch (IOException ioe) { 370 throw ioExceptionHandler(mConnection, ioe); 371 } finally { 372 destroyResponses(); 373 } 374 } 375 376 @Override 377 public void delete(boolean recurse) { 378 throw new Error("ImapStore.delete() not yet implemented"); 379 } 380 381 String[] getSearchUids(List<ImapResponse> responses) { 382 // S: * SEARCH 2 3 6 383 final ArrayList<String> uids = new ArrayList<String>(); 384 for (ImapResponse response : responses) { 385 if (!response.isDataResponse(0, ImapConstants.SEARCH)) { 386 continue; 387 } 388 // Found SEARCH response data 389 for (int i = 1; i < response.size(); i++) { 390 ImapString s = response.getStringOrEmpty(i); 391 if (s.isString()) { 392 uids.add(s.getString()); 393 } 394 } 395 } 396 return uids.toArray(Utility.EMPTY_STRINGS); 397 } 398 399 String[] searchForUids(String searchCriteria) throws MessagingException { 400 return searchForUids(searchCriteria, true); 401 } 402 403 /** 404 * I'm not a fan of having a parameter that determines whether to throw exceptions or 405 * consume them, but getMessage() for a date range needs to differentiate between 406 * a failure and just a legitimate empty result. 407 * See b/11183568. 408 * TODO: 409 * Either figure out how to make getMessage() with a date range work without this 410 * exception information, or make all users of searchForUids() handle the ImapException. 411 * It's too late in the release cycle to add this risk right now. 412 */ 413 @VisibleForTesting 414 String[] searchForUids(String searchCriteria, boolean swallowException) 415 throws MessagingException { 416 checkOpen(); 417 try { 418 try { 419 final String command = ImapConstants.UID_SEARCH + " " + searchCriteria; 420 final String[] result = getSearchUids(mConnection.executeSimpleCommand(command)); 421 LogUtils.d(Logging.LOG_TAG, "searchForUids '" + searchCriteria + "' results: " + 422 result.length); 423 return result; 424 } catch (ImapException me) { 425 LogUtils.d(Logging.LOG_TAG, me, "ImapException in search: " + searchCriteria); 426 if (swallowException) { 427 return Utility.EMPTY_STRINGS; // Not found 428 } else { 429 throw me; 430 } 431 } catch (IOException ioe) { 432 LogUtils.d(Logging.LOG_TAG, ioe, "IOException in search: " + searchCriteria); 433 throw ioExceptionHandler(mConnection, ioe); 434 } 435 } finally { 436 destroyResponses(); 437 } 438 } 439 440 @Override 441 @VisibleForTesting 442 public Message getMessage(String uid) throws MessagingException { 443 checkOpen(); 444 445 final String[] uids = searchForUids(ImapConstants.UID + " " + uid); 446 for (int i = 0; i < uids.length; i++) { 447 if (uids[i].equals(uid)) { 448 return new ImapMessage(uid, this); 449 } 450 } 451 return null; 452 } 453 454 @VisibleForTesting 455 protected static boolean isAsciiString(String str) { 456 int len = str.length(); 457 for (int i = 0; i < len; i++) { 458 char c = str.charAt(i); 459 if (c >= 128) return false; 460 } 461 return true; 462 } 463 464 /** 465 * Retrieve messages based on search parameters. We search FROM, TO, CC, SUBJECT, and BODY 466 * We send: SEARCH OR FROM "foo" (OR TO "foo" (OR CC "foo" (OR SUBJECT "foo" BODY "foo"))), but 467 * with the additional CHARSET argument and sending "foo" as a literal (e.g. {3}<CRLF>foo} 468 */ 469 @Override 470 @VisibleForTesting 471 public Message[] getMessages(SearchParams params, MessageRetrievalListener listener) 472 throws MessagingException { 473 List<String> commands = new ArrayList<String>(); 474 final String filter = params.mFilter; 475 // All servers MUST accept US-ASCII, so we'll send this as the CHARSET unless we're really 476 // dealing with a string that contains non-ascii characters 477 String charset = "US-ASCII"; 478 if (!isAsciiString(filter)) { 479 charset = "UTF-8"; 480 } 481 // This is the length of the string in octets (bytes), formatted as a string literal {n} 482 final String octetLength = "{" + filter.getBytes().length + "}"; 483 // Break the command up into pieces ending with the string literal length 484 commands.add(ImapConstants.UID_SEARCH + " CHARSET " + charset + " OR FROM " + octetLength); 485 commands.add(filter + " (OR TO " + octetLength); 486 commands.add(filter + " (OR CC " + octetLength); 487 commands.add(filter + " (OR SUBJECT " + octetLength); 488 commands.add(filter + " BODY " + octetLength); 489 commands.add(filter + ")))"); 490 return getMessagesInternal(complexSearchForUids(commands), listener); 491 } 492 493 /* package */ String[] complexSearchForUids(List<String> commands) throws MessagingException { 494 checkOpen(); 495 try { 496 try { 497 return getSearchUids(mConnection.executeComplexCommand(commands, false)); 498 } catch (ImapException e) { 499 return Utility.EMPTY_STRINGS; // not found; 500 } catch (IOException ioe) { 501 throw ioExceptionHandler(mConnection, ioe); 502 } 503 } finally { 504 destroyResponses(); 505 } 506 } 507 508 @Override 509 @VisibleForTesting 510 public Message[] getMessages(int start, int end, MessageRetrievalListener listener) 511 throws MessagingException { 512 if (start < 1 || end < 1 || end < start) { 513 throw new MessagingException(String.format("Invalid range: %d %d", start, end)); 514 } 515 LogUtils.d(Logging.LOG_TAG, "getMessages number " + start + " - " + end); 516 return getMessagesInternal( 517 searchForUids(String.format(Locale.US, "%d:%d NOT DELETED", start, end)), listener); 518 } 519 520 private String generateDateRangeCommand(final long startDate, final long endDate, 521 boolean useQuotes) 522 throws MessagingException { 523 // Dates must be formatted like: 7-Feb-1994. Time info within a date is not 524 // universally supported. 525 // XXX can I limit the maximum number of results? 526 final SimpleDateFormat formatter = new SimpleDateFormat("dd-LLL-yyyy", Locale.US); 527 formatter.setTimeZone(TimeZone.getTimeZone("UTC")); 528 final String sinceDateStr = formatter.format(endDate); 529 530 StringBuilder queryParam = new StringBuilder(); 531 queryParam.append( "1:* "); 532 // If the caller requests a startDate of zero, then ignore the BEFORE parameter. 533 // This makes sure that we can always query for the newest messages, even if our 534 // time is different from the imap server's time. 535 if (startDate != 0) { 536 final String beforeDateStr = formatter.format(startDate); 537 if (startDate < endDate) { 538 throw new MessagingException(String.format("Invalid date range: %s - %s", 539 sinceDateStr, beforeDateStr)); 540 } 541 queryParam.append("BEFORE "); 542 if (useQuotes) queryParam.append('\"'); 543 queryParam.append(beforeDateStr); 544 if (useQuotes) queryParam.append('\"'); 545 queryParam.append(" "); 546 } 547 queryParam.append("SINCE "); 548 if (useQuotes) queryParam.append('\"'); 549 queryParam.append(sinceDateStr); 550 if (useQuotes) queryParam.append('\"'); 551 552 return queryParam.toString(); 553 } 554 555 @Override 556 @VisibleForTesting 557 public Message[] getMessages(long startDate, long endDate, MessageRetrievalListener listener) 558 throws MessagingException { 559 String [] uids = null; 560 String command = generateDateRangeCommand(startDate, endDate, false); 561 LogUtils.d(Logging.LOG_TAG, "getMessages dateRange " + command.toString()); 562 563 try { 564 uids = searchForUids(command.toString(), false); 565 } catch (ImapException e) { 566 // TODO: This is a last minute hack to make certain servers work. Some servers 567 // demand that the date in the date range be surrounded by double quotes, other 568 // servers won't accept that. So if we can an ImapException using one method, 569 // try the other. 570 // See b/11183568 571 LogUtils.d(Logging.LOG_TAG, e, "query failed %s, trying alternate", 572 command.toString()); 573 command = generateDateRangeCommand(startDate, endDate, true); 574 try { 575 uids = searchForUids(command, true); 576 } catch (ImapException e2) { 577 LogUtils.w(Logging.LOG_TAG, e2, "query failed %s, fatal", command); 578 uids = null; 579 } 580 } 581 return getMessagesInternal(uids, listener); 582 } 583 584 @Override 585 @VisibleForTesting 586 public Message[] getMessages(String[] uids, MessageRetrievalListener listener) 587 throws MessagingException { 588 if (uids == null) { 589 uids = searchForUids("1:* NOT DELETED"); 590 } 591 return getMessagesInternal(uids, listener); 592 } 593 594 public Message[] getMessagesInternal(String[] uids, MessageRetrievalListener listener) { 595 final ArrayList<Message> messages = new ArrayList<Message>(uids.length); 596 for (int i = 0; i < uids.length; i++) { 597 final String uid = uids[i]; 598 final ImapMessage message = new ImapMessage(uid, this); 599 messages.add(message); 600 if (listener != null) { 601 listener.messageRetrieved(message); 602 } 603 } 604 return messages.toArray(Message.EMPTY_ARRAY); 605 } 606 607 @Override 608 public void fetch(Message[] messages, FetchProfile fp, MessageRetrievalListener listener) 609 throws MessagingException { 610 try { 611 fetchInternal(messages, fp, listener); 612 } catch (RuntimeException e) { // Probably a parser error. 613 LogUtils.w(Logging.LOG_TAG, "Exception detected: " + e.getMessage()); 614 if (mConnection != null) { 615 mConnection.logLastDiscourse(); 616 } 617 throw e; 618 } 619 } 620 621 public void fetchInternal(Message[] messages, FetchProfile fp, 622 MessageRetrievalListener listener) throws MessagingException { 623 if (messages.length == 0) { 624 return; 625 } 626 checkOpen(); 627 HashMap<String, Message> messageMap = new HashMap<String, Message>(); 628 for (Message m : messages) { 629 messageMap.put(m.getUid(), m); 630 } 631 632 /* 633 * Figure out what command we are going to run: 634 * FLAGS - UID FETCH (FLAGS) 635 * ENVELOPE - UID FETCH (INTERNALDATE UID RFC822.SIZE FLAGS BODY.PEEK[ 636 * HEADER.FIELDS (date subject from content-type to cc)]) 637 * STRUCTURE - UID FETCH (BODYSTRUCTURE) 638 * BODY_SANE - UID FETCH (BODY.PEEK[]<0.N>) where N = max bytes returned 639 * BODY - UID FETCH (BODY.PEEK[]) 640 * Part - UID FETCH (BODY.PEEK[ID]) where ID = mime part ID 641 */ 642 643 final LinkedHashSet<String> fetchFields = new LinkedHashSet<String>(); 644 645 fetchFields.add(ImapConstants.UID); 646 if (fp.contains(FetchProfile.Item.FLAGS)) { 647 fetchFields.add(ImapConstants.FLAGS); 648 } 649 if (fp.contains(FetchProfile.Item.ENVELOPE)) { 650 fetchFields.add(ImapConstants.INTERNALDATE); 651 fetchFields.add(ImapConstants.RFC822_SIZE); 652 fetchFields.add(ImapConstants.FETCH_FIELD_HEADERS); 653 } 654 if (fp.contains(FetchProfile.Item.STRUCTURE)) { 655 fetchFields.add(ImapConstants.BODYSTRUCTURE); 656 } 657 658 if (fp.contains(FetchProfile.Item.BODY_SANE)) { 659 fetchFields.add(ImapConstants.FETCH_FIELD_BODY_PEEK_SANE); 660 } 661 if (fp.contains(FetchProfile.Item.BODY)) { 662 fetchFields.add(ImapConstants.FETCH_FIELD_BODY_PEEK); 663 } 664 665 // TODO Why are we only fetching the first part given? 666 final Part fetchPart = fp.getFirstPart(); 667 if (fetchPart != null) { 668 final String[] partIds = 669 fetchPart.getHeader(MimeHeader.HEADER_ANDROID_ATTACHMENT_STORE_DATA); 670 // TODO Why can a single part have more than one Id? And why should we only fetch 671 // the first id if there are more than one? 672 if (partIds != null) { 673 fetchFields.add(ImapConstants.FETCH_FIELD_BODY_PEEK_BARE 674 + "[" + partIds[0] + "]"); 675 } 676 } 677 678 try { 679 mConnection.sendCommand(String.format(Locale.US, 680 ImapConstants.UID_FETCH + " %s (%s)", ImapStore.joinMessageUids(messages), 681 Utility.combine(fetchFields.toArray(new String[fetchFields.size()]), ' ') 682 ), false); 683 ImapResponse response; 684 do { 685 response = null; 686 try { 687 response = mConnection.readResponse(); 688 689 if (!response.isDataResponse(1, ImapConstants.FETCH)) { 690 continue; // Ignore 691 } 692 final ImapList fetchList = response.getListOrEmpty(2); 693 final String uid = fetchList.getKeyedStringOrEmpty(ImapConstants.UID) 694 .getString(); 695 if (TextUtils.isEmpty(uid)) continue; 696 697 ImapMessage message = (ImapMessage) messageMap.get(uid); 698 if (message == null) continue; 699 700 if (fp.contains(FetchProfile.Item.FLAGS)) { 701 final ImapList flags = 702 fetchList.getKeyedListOrEmpty(ImapConstants.FLAGS); 703 for (int i = 0, count = flags.size(); i < count; i++) { 704 final ImapString flag = flags.getStringOrEmpty(i); 705 if (flag.is(ImapConstants.FLAG_DELETED)) { 706 message.setFlagInternal(Flag.DELETED, true); 707 } else if (flag.is(ImapConstants.FLAG_ANSWERED)) { 708 message.setFlagInternal(Flag.ANSWERED, true); 709 } else if (flag.is(ImapConstants.FLAG_SEEN)) { 710 message.setFlagInternal(Flag.SEEN, true); 711 } else if (flag.is(ImapConstants.FLAG_FLAGGED)) { 712 message.setFlagInternal(Flag.FLAGGED, true); 713 } 714 } 715 } 716 if (fp.contains(FetchProfile.Item.ENVELOPE)) { 717 final Date internalDate = fetchList.getKeyedStringOrEmpty( 718 ImapConstants.INTERNALDATE).getDateOrNull(); 719 final int size = fetchList.getKeyedStringOrEmpty( 720 ImapConstants.RFC822_SIZE).getNumberOrZero(); 721 final String header = fetchList.getKeyedStringOrEmpty( 722 ImapConstants.BODY_BRACKET_HEADER, true).getString(); 723 724 message.setInternalDate(internalDate); 725 message.setSize(size); 726 message.parse(Utility.streamFromAsciiString(header)); 727 } 728 if (fp.contains(FetchProfile.Item.STRUCTURE)) { 729 ImapList bs = fetchList.getKeyedListOrEmpty( 730 ImapConstants.BODYSTRUCTURE); 731 if (!bs.isEmpty()) { 732 try { 733 parseBodyStructure(bs, message, ImapConstants.TEXT); 734 } catch (MessagingException e) { 735 if (Logging.LOGD) { 736 LogUtils.v(Logging.LOG_TAG, e, "Error handling message"); 737 } 738 message.setBody(null); 739 } 740 } 741 } 742 if (fp.contains(FetchProfile.Item.BODY) 743 || fp.contains(FetchProfile.Item.BODY_SANE)) { 744 // Body is keyed by "BODY[]...". 745 // Previously used "BODY[..." but this can be confused with "BODY[HEADER..." 746 // TODO Should we accept "RFC822" as well?? 747 ImapString body = fetchList.getKeyedStringOrEmpty("BODY[]", true); 748 InputStream bodyStream = body.getAsStream(); 749 message.parse(bodyStream); 750 } 751 if (fetchPart != null) { 752 InputStream bodyStream = 753 fetchList.getKeyedStringOrEmpty("BODY[", true).getAsStream(); 754 String encodings[] = fetchPart.getHeader( 755 MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING); 756 757 String contentTransferEncoding = null; 758 if (encodings != null && encodings.length > 0) { 759 contentTransferEncoding = encodings[0]; 760 } else { 761 // According to http://tools.ietf.org/html/rfc2045#section-6.1 762 // "7bit" is the default. 763 contentTransferEncoding = "7bit"; 764 } 765 766 try { 767 // TODO Don't create 2 temp files. 768 // decodeBody creates BinaryTempFileBody, but we could avoid this 769 // if we implement ImapStringBody. 770 // (We'll need to share a temp file. Protect it with a ref-count.) 771 fetchPart.setBody(decodeBody(bodyStream, contentTransferEncoding, 772 fetchPart.getSize(), listener)); 773 } catch(Exception e) { 774 // TODO: Figure out what kinds of exceptions might actually be thrown 775 // from here. This blanket catch-all is because we're not sure what to 776 // do if we don't have a contentTransferEncoding, and we don't have 777 // time to figure out what exceptions might be thrown. 778 LogUtils.e(Logging.LOG_TAG, "Error fetching body %s", e); 779 } 780 } 781 782 if (listener != null) { 783 listener.messageRetrieved(message); 784 } 785 } finally { 786 destroyResponses(); 787 } 788 } while (!response.isTagged()); 789 } catch (IOException ioe) { 790 throw ioExceptionHandler(mConnection, ioe); 791 } 792 } 793 794 /** 795 * Removes any content transfer encoding from the stream and returns a Body. 796 * This code is taken/condensed from MimeUtility.decodeBody 797 */ 798 private static Body decodeBody(InputStream in, String contentTransferEncoding, int size, 799 MessageRetrievalListener listener) throws IOException { 800 // Get a properly wrapped input stream 801 in = MimeUtility.getInputStreamForContentTransferEncoding(in, contentTransferEncoding); 802 BinaryTempFileBody tempBody = new BinaryTempFileBody(); 803 OutputStream out = tempBody.getOutputStream(); 804 try { 805 byte[] buffer = new byte[COPY_BUFFER_SIZE]; 806 int n = 0; 807 int count = 0; 808 while (-1 != (n = in.read(buffer))) { 809 out.write(buffer, 0, n); 810 count += n; 811 if (listener != null) { 812 if (size == 0) { 813 // We don't know how big the file is, so just fake it. 814 listener.loadAttachmentProgress((int)Math.ceil(100 * (1-1.0/count))); 815 } else { 816 listener.loadAttachmentProgress(count * 100 / size); 817 } 818 } 819 } 820 } catch (Base64DataException bde) { 821 String warning = "\n\n" + ImapService.getMessageDecodeErrorString(); 822 out.write(warning.getBytes()); 823 } finally { 824 out.close(); 825 } 826 return tempBody; 827 } 828 829 @Override 830 public Flag[] getPermanentFlags() { 831 return PERMANENT_FLAGS; 832 } 833 834 /** 835 * Handle any untagged responses that the caller doesn't care to handle themselves. 836 * @param responses 837 */ 838 private void handleUntaggedResponses(List<ImapResponse> responses) { 839 for (ImapResponse response : responses) { 840 handleUntaggedResponse(response); 841 } 842 } 843 844 /** 845 * Handle an untagged response that the caller doesn't care to handle themselves. 846 * @param response 847 */ 848 private void handleUntaggedResponse(ImapResponse response) { 849 if (response.isDataResponse(1, ImapConstants.EXISTS)) { 850 mMessageCount = response.getStringOrEmpty(0).getNumberOrZero(); 851 } 852 } 853 854 private static void parseBodyStructure(ImapList bs, Part part, String id) 855 throws MessagingException { 856 if (bs.getElementOrNone(0).isList()) { 857 /* 858 * This is a multipart/* 859 */ 860 MimeMultipart mp = new MimeMultipart(); 861 for (int i = 0, count = bs.size(); i < count; i++) { 862 ImapElement e = bs.getElementOrNone(i); 863 if (e.isList()) { 864 /* 865 * For each part in the message we're going to add a new BodyPart and parse 866 * into it. 867 */ 868 MimeBodyPart bp = new MimeBodyPart(); 869 if (id.equals(ImapConstants.TEXT)) { 870 parseBodyStructure(bs.getListOrEmpty(i), bp, Integer.toString(i + 1)); 871 872 } else { 873 parseBodyStructure(bs.getListOrEmpty(i), bp, id + "." + (i + 1)); 874 } 875 mp.addBodyPart(bp); 876 877 } else { 878 if (e.isString()) { 879 mp.setSubType(bs.getStringOrEmpty(i).getString().toLowerCase(Locale.US)); 880 } 881 break; // Ignore the rest of the list. 882 } 883 } 884 part.setBody(mp); 885 } else { 886 /* 887 * This is a body. We need to add as much information as we can find out about 888 * it to the Part. 889 */ 890 891 /* 892 body type 893 body subtype 894 body parameter parenthesized list 895 body id 896 body description 897 body encoding 898 body size 899 */ 900 901 final ImapString type = bs.getStringOrEmpty(0); 902 final ImapString subType = bs.getStringOrEmpty(1); 903 final String mimeType = 904 (type.getString() + "/" + subType.getString()).toLowerCase(Locale.US); 905 906 final ImapList bodyParams = bs.getListOrEmpty(2); 907 final ImapString cid = bs.getStringOrEmpty(3); 908 final ImapString encoding = bs.getStringOrEmpty(5); 909 final int size = bs.getStringOrEmpty(6).getNumberOrZero(); 910 911 if (MimeUtility.mimeTypeMatches(mimeType, MimeUtility.MIME_TYPE_RFC822)) { 912 // A body type of type MESSAGE and subtype RFC822 913 // contains, immediately after the basic fields, the 914 // envelope structure, body structure, and size in 915 // text lines of the encapsulated message. 916 // [MESSAGE, RFC822, [NAME, filename.eml], NIL, NIL, 7BIT, 5974, NIL, 917 // [INLINE, [FILENAME*0, Fwd: Xxx..., FILENAME*1, filename.eml]], NIL] 918 /* 919 * This will be caught by fetch and handled appropriately. 920 */ 921 throw new MessagingException("BODYSTRUCTURE " + MimeUtility.MIME_TYPE_RFC822 922 + " not yet supported."); 923 } 924 925 /* 926 * Set the content type with as much information as we know right now. 927 */ 928 final StringBuilder contentType = new StringBuilder(mimeType); 929 930 /* 931 * If there are body params we might be able to get some more information out 932 * of them. 933 */ 934 for (int i = 1, count = bodyParams.size(); i < count; i += 2) { 935 936 // TODO We need to convert " into %22, but 937 // because MimeUtility.getHeaderParameter doesn't recognize it, 938 // we can't fix it for now. 939 contentType.append(String.format(";\n %s=\"%s\"", 940 bodyParams.getStringOrEmpty(i - 1).getString(), 941 bodyParams.getStringOrEmpty(i).getString())); 942 } 943 944 part.setHeader(MimeHeader.HEADER_CONTENT_TYPE, contentType.toString()); 945 946 // Extension items 947 final ImapList bodyDisposition; 948 949 if (type.is(ImapConstants.TEXT) && bs.getElementOrNone(9).isList()) { 950 // If media-type is TEXT, 9th element might be: [body-fld-lines] := number 951 // So, if it's not a list, use 10th element. 952 // (Couldn't find evidence in the RFC if it's ALWAYS 10th element.) 953 bodyDisposition = bs.getListOrEmpty(9); 954 } else { 955 bodyDisposition = bs.getListOrEmpty(8); 956 } 957 958 final StringBuilder contentDisposition = new StringBuilder(); 959 960 if (bodyDisposition.size() > 0) { 961 final String bodyDisposition0Str = 962 bodyDisposition.getStringOrEmpty(0).getString().toLowerCase(Locale.US); 963 if (!TextUtils.isEmpty(bodyDisposition0Str)) { 964 contentDisposition.append(bodyDisposition0Str); 965 } 966 967 final ImapList bodyDispositionParams = bodyDisposition.getListOrEmpty(1); 968 if (!bodyDispositionParams.isEmpty()) { 969 /* 970 * If there is body disposition information we can pull some more 971 * information about the attachment out. 972 */ 973 for (int i = 1, count = bodyDispositionParams.size(); i < count; i += 2) { 974 975 // TODO We need to convert " into %22. See above. 976 contentDisposition.append(String.format(Locale.US, ";\n %s=\"%s\"", 977 bodyDispositionParams.getStringOrEmpty(i - 1) 978 .getString().toLowerCase(Locale.US), 979 bodyDispositionParams.getStringOrEmpty(i).getString())); 980 } 981 } 982 } 983 984 if ((size > 0) 985 && (MimeUtility.getHeaderParameter(contentDisposition.toString(), "size") 986 == null)) { 987 contentDisposition.append(String.format(Locale.US, ";\n size=%d", size)); 988 } 989 990 if (contentDisposition.length() > 0) { 991 /* 992 * Set the content disposition containing at least the size. Attachment 993 * handling code will use this down the road. 994 */ 995 part.setHeader(MimeHeader.HEADER_CONTENT_DISPOSITION, 996 contentDisposition.toString()); 997 } 998 999 /* 1000 * Set the Content-Transfer-Encoding header. Attachment code will use this 1001 * to parse the body. 1002 */ 1003 if (!encoding.isEmpty()) { 1004 part.setHeader(MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING, 1005 encoding.getString()); 1006 } 1007 1008 /* 1009 * Set the Content-ID header. 1010 */ 1011 if (!cid.isEmpty()) { 1012 part.setHeader(MimeHeader.HEADER_CONTENT_ID, cid.getString()); 1013 } 1014 1015 if (size > 0) { 1016 if (part instanceof ImapMessage) { 1017 ((ImapMessage) part).setSize(size); 1018 } else if (part instanceof MimeBodyPart) { 1019 ((MimeBodyPart) part).setSize(size); 1020 } else { 1021 throw new MessagingException("Unknown part type " + part.toString()); 1022 } 1023 } 1024 part.setHeader(MimeHeader.HEADER_ANDROID_ATTACHMENT_STORE_DATA, id); 1025 } 1026 1027 } 1028 1029 /** 1030 * Appends the given messages to the selected folder. This implementation also determines 1031 * the new UID of the given message on the IMAP server and sets the Message's UID to the 1032 * new server UID. 1033 * @param message Message 1034 * @param noTimeout Set to true on manual syncs, disables the timeout after sending the message 1035 * content to the server 1036 */ 1037 @Override 1038 public void appendMessage(final Context context, final Message message, final boolean noTimeout) 1039 throws MessagingException { 1040 checkOpen(); 1041 try { 1042 // Create temp file 1043 /** 1044 * We need to know the encoded message size before we upload it, and encoding 1045 * attachments as Base64, possibly reading from a slow provider, is a non-trivial 1046 * operation. So we write the contents to a temp file while measuring the size, 1047 * and then use that temp file and size to do the actual upsync. 1048 * For context, most classic email clients would store the message in RFC822 format 1049 * internally, and so would not need to do this on-the-fly. 1050 */ 1051 final File tempDir = context.getExternalCacheDir(); 1052 final File tempFile = File.createTempFile("IMAPupsync", ".eml", tempDir); 1053 // Delete here so we don't leave the file lingering. We've got a handle to it so we 1054 // can still use it. 1055 final boolean deleteSuccessful = tempFile.delete(); 1056 if (!deleteSuccessful) { 1057 LogUtils.w(LogUtils.TAG, "Could not delete temp file %s", 1058 tempFile.getAbsolutePath()); 1059 } 1060 final OutputStream tempOut = new FileOutputStream(tempFile); 1061 // Create output count while writing temp file 1062 final CountingOutputStream out = new CountingOutputStream(tempOut); 1063 final EOLConvertingOutputStream eolOut = new EOLConvertingOutputStream(out); 1064 message.writeTo(eolOut); 1065 eolOut.flush(); 1066 // Create flag list (most often this will be "\SEEN") 1067 String flagList = ""; 1068 Flag[] flags = message.getFlags(); 1069 if (flags.length > 0) { 1070 StringBuilder sb = new StringBuilder(); 1071 for (final Flag flag : flags) { 1072 if (flag == Flag.SEEN) { 1073 sb.append(" " + ImapConstants.FLAG_SEEN); 1074 } else if (flag == Flag.FLAGGED) { 1075 sb.append(" " + ImapConstants.FLAG_FLAGGED); 1076 } 1077 } 1078 if (sb.length() > 0) { 1079 flagList = sb.substring(1); 1080 } 1081 } 1082 1083 mConnection.sendCommand( 1084 String.format(Locale.US, ImapConstants.APPEND + " \"%s\" (%s) {%d}", 1085 ImapStore.encodeFolderName(mName, mStore.mPathPrefix), 1086 flagList, 1087 out.getCount()), false); 1088 ImapResponse response; 1089 do { 1090 final int socketTimeout = mConnection.mTransport.getSoTimeout(); 1091 try { 1092 // Need to set the timeout to unlimited since we might be upsyncing a pretty 1093 // big attachment so who knows how long it'll take. It would sure be nice 1094 // if this only timed out after the send buffer drained but welp. 1095 if (noTimeout) { 1096 // For now, only unset the timeout if we're doing a manual sync 1097 mConnection.mTransport.setSoTimeout(0); 1098 } 1099 response = mConnection.readResponse(); 1100 if (response.isContinuationRequest()) { 1101 final OutputStream transportOutputStream = 1102 mConnection.mTransport.getOutputStream(); 1103 IOUtils.copyLarge(new FileInputStream(tempFile), transportOutputStream); 1104 transportOutputStream.write('\r'); 1105 transportOutputStream.write('\n'); 1106 transportOutputStream.flush(); 1107 } else if (!response.isTagged()) { 1108 handleUntaggedResponse(response); 1109 } 1110 } finally { 1111 mConnection.mTransport.setSoTimeout(socketTimeout); 1112 } 1113 } while (!response.isTagged()); 1114 1115 // TODO Why not check the response? 1116 1117 /* 1118 * Try to recover the UID of the message from an APPENDUID response. 1119 * e.g. 11 OK [APPENDUID 2 238268] APPEND completed 1120 */ 1121 final ImapList appendList = response.getListOrEmpty(1); 1122 if ((appendList.size() >= 3) && appendList.is(0, ImapConstants.APPENDUID)) { 1123 String serverUid = appendList.getStringOrEmpty(2).getString(); 1124 if (!TextUtils.isEmpty(serverUid)) { 1125 message.setUid(serverUid); 1126 return; 1127 } 1128 } 1129 1130 /* 1131 * Try to find the UID of the message we just appended using the 1132 * Message-ID header. If there are more than one response, take the 1133 * last one, as it's most likely the newest (the one we just uploaded). 1134 */ 1135 final String messageId = message.getMessageId(); 1136 if (messageId == null || messageId.length() == 0) { 1137 return; 1138 } 1139 // Most servers don't care about parenthesis in the search query [and, some 1140 // fail to work if they are used] 1141 String[] uids = searchForUids( 1142 String.format(Locale.US, "HEADER MESSAGE-ID %s", messageId)); 1143 if (uids.length > 0) { 1144 message.setUid(uids[0]); 1145 } 1146 // However, there's at least one server [AOL] that fails to work unless there 1147 // are parenthesis, so, try this as a last resort 1148 uids = searchForUids(String.format(Locale.US, "(HEADER MESSAGE-ID %s)", messageId)); 1149 if (uids.length > 0) { 1150 message.setUid(uids[0]); 1151 } 1152 } catch (IOException ioe) { 1153 throw ioExceptionHandler(mConnection, ioe); 1154 } finally { 1155 destroyResponses(); 1156 } 1157 } 1158 1159 @Override 1160 public Message[] expunge() throws MessagingException { 1161 checkOpen(); 1162 try { 1163 handleUntaggedResponses(mConnection.executeSimpleCommand(ImapConstants.EXPUNGE)); 1164 } catch (IOException ioe) { 1165 throw ioExceptionHandler(mConnection, ioe); 1166 } finally { 1167 destroyResponses(); 1168 } 1169 return null; 1170 } 1171 1172 @Override 1173 public void setFlags(Message[] messages, Flag[] flags, boolean value) 1174 throws MessagingException { 1175 checkOpen(); 1176 1177 String allFlags = ""; 1178 if (flags.length > 0) { 1179 StringBuilder flagList = new StringBuilder(); 1180 for (int i = 0, count = flags.length; i < count; i++) { 1181 Flag flag = flags[i]; 1182 if (flag == Flag.SEEN) { 1183 flagList.append(" " + ImapConstants.FLAG_SEEN); 1184 } else if (flag == Flag.DELETED) { 1185 flagList.append(" " + ImapConstants.FLAG_DELETED); 1186 } else if (flag == Flag.FLAGGED) { 1187 flagList.append(" " + ImapConstants.FLAG_FLAGGED); 1188 } else if (flag == Flag.ANSWERED) { 1189 flagList.append(" " + ImapConstants.FLAG_ANSWERED); 1190 } 1191 } 1192 allFlags = flagList.substring(1); 1193 } 1194 try { 1195 mConnection.executeSimpleCommand(String.format(Locale.US, 1196 ImapConstants.UID_STORE + " %s %s" + ImapConstants.FLAGS_SILENT + " (%s)", 1197 ImapStore.joinMessageUids(messages), 1198 value ? "+" : "-", 1199 allFlags)); 1200 1201 } catch (IOException ioe) { 1202 throw ioExceptionHandler(mConnection, ioe); 1203 } finally { 1204 destroyResponses(); 1205 } 1206 } 1207 1208 /** 1209 * Persists this folder. We will always perform the proper database operation (e.g. 1210 * 'save' or 'update'). As an optimization, if a folder has not been modified, no 1211 * database operations are performed. 1212 */ 1213 void save(Context context) { 1214 final Mailbox mailbox = mMailbox; 1215 if (!mailbox.isSaved()) { 1216 mailbox.save(context); 1217 mHash = mailbox.getHashes(); 1218 } else { 1219 Object[] hash = mailbox.getHashes(); 1220 if (!Arrays.equals(mHash, hash)) { 1221 mailbox.update(context, mailbox.toContentValues()); 1222 mHash = hash; // Save updated hash 1223 } 1224 } 1225 } 1226 1227 /** 1228 * Selects the folder for use. Before performing any operations on this folder, it 1229 * must be selected. 1230 */ 1231 private void doSelect() throws IOException, MessagingException { 1232 final List<ImapResponse> responses = mConnection.executeSimpleCommand( 1233 String.format(Locale.US, ImapConstants.SELECT + " \"%s\"", 1234 ImapStore.encodeFolderName(mName, mStore.mPathPrefix))); 1235 1236 // Assume the folder is opened read-write; unless we are notified otherwise 1237 mMode = OpenMode.READ_WRITE; 1238 int messageCount = -1; 1239 for (ImapResponse response : responses) { 1240 if (response.isDataResponse(1, ImapConstants.EXISTS)) { 1241 messageCount = response.getStringOrEmpty(0).getNumberOrZero(); 1242 } else if (response.isOk()) { 1243 final ImapString responseCode = response.getResponseCodeOrEmpty(); 1244 if (responseCode.is(ImapConstants.READ_ONLY)) { 1245 mMode = OpenMode.READ_ONLY; 1246 } else if (responseCode.is(ImapConstants.READ_WRITE)) { 1247 mMode = OpenMode.READ_WRITE; 1248 } 1249 } else if (response.isTagged()) { // Not OK 1250 throw new MessagingException("Can't open mailbox: " 1251 + response.getStatusResponseTextOrEmpty()); 1252 } 1253 } 1254 if (messageCount == -1) { 1255 throw new MessagingException("Did not find message count during select"); 1256 } 1257 mMessageCount = messageCount; 1258 mExists = true; 1259 } 1260 1261 private void checkOpen() throws MessagingException { 1262 if (!isOpen()) { 1263 throw new MessagingException("Folder " + mName + " is not open."); 1264 } 1265 } 1266 1267 private MessagingException ioExceptionHandler(ImapConnection connection, IOException ioe) { 1268 if (DebugUtils.DEBUG) { 1269 LogUtils.d(Logging.LOG_TAG, "IO Exception detected: ", ioe); 1270 } 1271 connection.close(); 1272 if (connection == mConnection) { 1273 mConnection = null; // To prevent close() from returning the connection to the pool. 1274 close(false); 1275 } 1276 return new MessagingException(MessagingException.IOERROR, "IO Error", ioe); 1277 } 1278 1279 @Override 1280 public boolean equals(Object o) { 1281 if (o instanceof ImapFolder) { 1282 return ((ImapFolder)o).mName.equals(mName); 1283 } 1284 return super.equals(o); 1285 } 1286 1287 @Override 1288 public Message createMessage(String uid) { 1289 return new ImapMessage(uid, this); 1290 } 1291} 1292