sqlite3_android.cpp revision 76b3da43e488769ceaae0698e5e23cfde84b4ce2 
1/*
2 * Copyright (C) 2007 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define LOG_TAG "sqlite3_android"
18
19#include <ctype.h>
20#include <stdlib.h>
21#include <string.h>
22#include <unistd.h>
23
24#include <unicode/ucol.h>
25#include <unicode/uiter.h>
26#include <unicode/ustring.h>
27#include <unicode/utypes.h>
28#include <cutils/log.h>
29
30#include "sqlite3_android.h"
31#include "PhoneNumberUtils.h"
32#include "PhonebookIndex.h"
33#include "PhoneticStringUtils.h"
34
35#define ENABLE_ANDROID_LOG 0
36#define SMALL_BUFFER_SIZE 10
37
38static int collate16(void *p, int n1, const void *v1, int n2, const void *v2)
39{
40    UCollator *coll = (UCollator *) p;
41    UCollationResult result = ucol_strcoll(coll, (const UChar *) v1, n1,
42                                                 (const UChar *) v2, n2);
43
44    if (result == UCOL_LESS) {
45        return -1;
46    } else if (result == UCOL_GREATER) {
47        return 1;
48    } else {
49        return 0;
50    }
51}
52
53static int collate8(void *p, int n1, const void *v1, int n2, const void *v2)
54{
55    UCollator *coll = (UCollator *) p;
56    UCharIterator i1, i2;
57    UErrorCode status = U_ZERO_ERROR;
58
59    uiter_setUTF8(&i1, (const char *) v1, n1);
60    uiter_setUTF8(&i2, (const char *) v2, n2);
61
62    UCollationResult result = ucol_strcollIter(coll, &i1, &i2, &status);
63
64    if (U_FAILURE(status)) {
65//        LOGE("Collation iterator error: %d\n", status);
66    }
67
68    if (result == UCOL_LESS) {
69        return -1;
70    } else if (result == UCOL_GREATER) {
71        return 1;
72    } else {
73        return 0;
74    }
75}
76
77/**
78 * Obtains the first UNICODE letter from the supplied string, normalizes and returns it.
79 */
80static void get_phonebook_index(
81    sqlite3_context * context, int argc, sqlite3_value ** argv)
82{
83    if (argc != 2) {
84      sqlite3_result_null(context);
85      return;
86    }
87
88    char const * src = (char const *)sqlite3_value_text(argv[0]);
89    char const * locale = (char const *)sqlite3_value_text(argv[1]);
90    if (src == NULL || src[0] == 0 || locale == NULL) {
91      sqlite3_result_null(context);
92      return;
93    }
94
95    UCharIterator iter;
96    uiter_setUTF8(&iter, src, -1);
97
98    UBool isError = FALSE;
99    UChar index[SMALL_BUFFER_SIZE];
100    uint32_t len = android::GetPhonebookIndex(&iter, locale, index, sizeof(index), &isError);
101    if (isError) {
102      sqlite3_result_null(context);
103      return;
104    }
105
106    uint32_t outlen = 0;
107    uint8_t out[SMALL_BUFFER_SIZE];
108    for (uint32_t i = 0; i < len; i++) {
109      U8_APPEND(out, outlen, sizeof(out), index[i], isError);
110      if (isError) {
111        sqlite3_result_null(context);
112        return;
113      }
114    }
115
116    if (outlen == 0) {
117      sqlite3_result_null(context);
118      return;
119    }
120
121    sqlite3_result_text(context, (const char*)out, outlen, SQLITE_TRANSIENT);
122}
123
124static void get_phonetically_sortable_string(
125    sqlite3_context * context, int argc, sqlite3_value ** argv)
126{
127    if (argc != 1) {
128      sqlite3_result_null(context);
129      return;
130    }
131    char const * src = (char const *)sqlite3_value_text(argv[0]);
132    char * ret;
133    size_t len;
134
135    if (!android::GetPhoneticallySortableString(src, &ret, &len)) {
136        // Put this text at the end of a list.
137        sqlite3_result_text(context, "\xF0\x9F\xBF\xBD", -1, SQLITE_STATIC);
138        // sqlite3_result_null(context);
139    } else {
140        sqlite3_result_text(context, ret, len, free);
141    }
142}
143
144static void get_normalized_string(
145    sqlite3_context * context, int argc, sqlite3_value ** argv)
146{
147    if (argc != 1) {
148      sqlite3_result_null(context);
149      return;
150    }
151    char const * src = (char const *)sqlite3_value_text(argv[0]);
152    char * ret;
153    size_t len;
154
155    if (!android::GetNormalizedString(src, &ret, &len)) {
156        // Probably broken string. Return 0 length string.
157        sqlite3_result_text(context, "", -1, SQLITE_STATIC);
158    } else {
159        sqlite3_result_text(context, ret, len, free);
160    }
161}
162
163static void phone_numbers_equal(sqlite3_context * context, int argc, sqlite3_value ** argv)
164{
165    if (argc != 2 && argc != 3) {
166        sqlite3_result_int(context, 0);
167        return;
168    }
169
170    char const * num1 = (char const *)sqlite3_value_text(argv[0]);
171    char const * num2 = (char const *)sqlite3_value_text(argv[1]);
172
173    bool use_strict = false;
174    if (argc == 3) {
175        use_strict = (sqlite3_value_int(argv[2]) != 0);
176    }
177
178    if (num1 == NULL || num2 == NULL) {
179        sqlite3_result_null(context);
180        return;
181    }
182
183    bool equal =
184        (use_strict ?
185         android::phone_number_compare_strict(num1, num2) :
186         android::phone_number_compare_loose(num1, num2));
187
188    if (equal) {
189        sqlite3_result_int(context, 1);
190    } else {
191        sqlite3_result_int(context, 0);
192    }
193}
194
195#if ENABLE_ANDROID_LOG
196static void android_log(sqlite3_context * context, int argc, sqlite3_value ** argv)
197{
198    char const * tag = "sqlite_trigger";
199    char const * msg = "";
200    int msgIndex = 0;
201
202    switch (argc) {
203        case 2:
204            tag = (char const *)sqlite3_value_text(argv[0]);
205            if (tag == NULL) {
206                tag = "sqlite_trigger";
207            }
208            msgIndex = 1;
209        case 1:
210            msg = (char const *)sqlite3_value_text(argv[msgIndex]);
211            if (msg == NULL) {
212                msg = "";
213            }
214            LOG(LOG_INFO, tag, msg);
215            sqlite3_result_int(context, 1);
216            return;
217
218        default:
219            sqlite3_result_int(context, 0);
220            return;
221    }
222}
223#endif
224
225static void delete_file(sqlite3_context * context, int argc, sqlite3_value ** argv)
226{
227    if (argc != 1) {
228        sqlite3_result_int(context, 0);
229        return;
230    }
231
232    char const * path = (char const *)sqlite3_value_text(argv[0]);
233    char const * external_storage = getenv("EXTERNAL_STORAGE");
234    if (path == NULL || external_storage == NULL) {
235        sqlite3_result_null(context);
236        return;
237    }
238
239    if (strncmp(external_storage, path, strlen(external_storage)) != 0) {
240        sqlite3_result_null(context);
241        return;
242    }
243    if (strstr(path, "/../") != NULL) {
244        sqlite3_result_null(context);
245        return;
246    }
247
248    int err = unlink(path);
249    if (err != -1) {
250        // No error occured, return true
251        sqlite3_result_int(context, 1);
252    } else {
253        // An error occured, return false
254        sqlite3_result_int(context, 0);
255    }
256}
257
258static void tokenize_auxdata_delete(void * data)
259{
260    sqlite3_stmt * statement = (sqlite3_stmt *)data;
261    sqlite3_finalize(statement);
262}
263
264static void base16Encode(char* dest, const char* src, uint32_t size)
265{
266    static const char * BASE16_TABLE = "0123456789abcdef";
267    for (uint32_t i = 0; i < size; i++) {
268        char ch = *src++;
269        *dest++ = BASE16_TABLE[ (ch & 0xf0) >> 4 ];
270        *dest++ = BASE16_TABLE[ (ch & 0x0f)      ];
271    }
272}
273
274struct SqliteUserData {
275    sqlite3 * handle;
276    UCollator* collator;
277};
278
279/**
280 * This function is invoked as:
281 *
282 *  _TOKENIZE('<token_table>', <data_row_id>, <data>, <delimiter>,
283 *             <use_token_index>, <data_tag>)
284 *
285 * If <use_token_index> is omitted, it is treated as 0.
286 * If <data_tag> is omitted, it is treated as NULL.
287 *
288 * It will split <data> on each instance of <delimiter> and insert each token
289 * into <token_table>. The following columns in <token_table> are used:
290 * token TEXT, source INTEGER, token_index INTEGER, tag (any type)
291 * The token_index column is not required if <use_token_index> is 0.
292 * The tag column is not required if <data_tag> is NULL.
293 *
294 * One row is inserted for each token in <data>.
295 * In each inserted row, 'source' is <data_row_id>.
296 * In the first inserted row, 'token' is the hex collation key of
297 * the entire <data> string, and 'token_index' is 0.
298 * In each row I (where 1 <= I < N, and N is the number of tokens in <data>)
299 * 'token' will be set to the hex collation key of the I:th token (0-based).
300 * If <use_token_index> != 0, 'token_index' is set to I.
301 * If <data_tag> is not NULL, 'tag' is set to <data_tag>.
302 *
303 * In other words, there will be one row for the entire string,
304 * and one row for each token except the first one.
305 *
306 * The function returns the number of tokens generated.
307 */
308static void tokenize(sqlite3_context * context, int argc, sqlite3_value ** argv)
309{
310    //LOGD("enter tokenize");
311    int err;
312    int useTokenIndex = 0;
313    int useDataTag = 0;
314
315    if (!(argc >= 4 || argc <= 6)) {
316        LOGE("Tokenize requires 4 to 6 arguments");
317        sqlite3_result_null(context);
318        return;
319    }
320
321    if (argc > 4) {
322        useTokenIndex = sqlite3_value_int(argv[4]);
323    }
324
325    if (argc > 5) {
326        useDataTag = (sqlite3_value_type(argv[5]) != SQLITE_NULL);
327    }
328
329    sqlite3 * handle = sqlite3_context_db_handle(context);
330    UCollator* collator = (UCollator*)sqlite3_user_data(context);
331    char const * tokenTable = (char const *)sqlite3_value_text(argv[0]);
332    if (tokenTable == NULL) {
333        LOGE("tokenTable null");
334        sqlite3_result_null(context);
335        return;
336    }
337
338    // Get or create the prepared statement for the insertions
339    sqlite3_stmt * statement = (sqlite3_stmt *)sqlite3_get_auxdata(context, 0);
340    if (!statement) {
341        char const * tokenIndexCol = useTokenIndex ? ", token_index" : "";
342        char const * tokenIndexParam = useTokenIndex ? ", ?" : "";
343        char const * dataTagCol = useDataTag ? ", tag" : "";
344        char const * dataTagParam = useDataTag ? ", ?" : "";
345        char * sql = sqlite3_mprintf("INSERT INTO %s (token, source%s%s) VALUES (?, ?%s%s);",
346                tokenTable, tokenIndexCol, dataTagCol, tokenIndexParam, dataTagParam);
347        err = sqlite3_prepare_v2(handle, sql, -1, &statement, NULL);
348        sqlite3_free(sql);
349        if (err) {
350            LOGE("prepare failed");
351            sqlite3_result_null(context);
352            return;
353        }
354        // This binds the statement to the table it was compiled against, which is argv[0].
355        // If this function is ever called with a different table the finalizer will be called
356        // and sqlite3_get_auxdata() will return null above, forcing a recompile for the new table.
357        sqlite3_set_auxdata(context, 0, statement, tokenize_auxdata_delete);
358    } else {
359        // Reset the cached statement so that binding the row ID will work properly
360        sqlite3_reset(statement);
361    }
362
363    // Bind the row ID of the source row
364    int64_t rowID = sqlite3_value_int64(argv[1]);
365    err = sqlite3_bind_int64(statement, 2, rowID);
366    if (err != SQLITE_OK) {
367        LOGE("bind failed");
368        sqlite3_result_null(context);
369        return;
370    }
371
372    // Bind <data_tag> to the tag column
373    if (useDataTag) {
374        int dataTagParamIndex = useTokenIndex ? 4 : 3;
375        err = sqlite3_bind_value(statement, dataTagParamIndex, argv[5]);
376        if (err != SQLITE_OK) {
377            LOGE("bind failed");
378            sqlite3_result_null(context);
379            return;
380        }
381    }
382
383    // Get the raw bytes for the string to tokenize
384    // the string will be modified by following code
385    // however, sqlite did not reuse the string, so it is safe to not dup it
386    UChar * origData = (UChar *)sqlite3_value_text16(argv[2]);
387    if (origData == NULL) {
388        sqlite3_result_null(context);
389        return;
390    }
391
392    // Get the raw bytes for the delimiter
393    const UChar * delim = (const UChar *)sqlite3_value_text16(argv[3]);
394    if (delim == NULL) {
395        LOGE("can't get delimiter");
396        sqlite3_result_null(context);
397        return;
398    }
399
400    UChar * token = NULL;
401    UChar *state;
402    int numTokens = 0;
403
404    do {
405        if (numTokens == 0) {
406            token = origData;
407        }
408
409        // Reset the program so we can use it to perform the insert
410        sqlite3_reset(statement);
411        UErrorCode status = U_ZERO_ERROR;
412        char keybuf[1024];
413        uint32_t result = ucol_getSortKey(collator, token, -1, (uint8_t*)keybuf, sizeof(keybuf)-1);
414        if (result > sizeof(keybuf)) {
415            // TODO allocate memory for this super big string
416            LOGE("ucol_getSortKey needs bigger buffer %d", result);
417            break;
418        }
419        uint32_t keysize = result-1;
420        uint32_t base16Size = keysize*2;
421        char *base16buf = (char*)malloc(base16Size);
422        base16Encode(base16buf, keybuf, keysize);
423        err = sqlite3_bind_text(statement, 1, base16buf, base16Size, SQLITE_STATIC);
424
425        if (err != SQLITE_OK) {
426            LOGE(" sqlite3_bind_text16 error %d", err);
427            free(base16buf);
428            break;
429        }
430
431        if (useTokenIndex) {
432            err = sqlite3_bind_int(statement, 3, numTokens);
433            if (err != SQLITE_OK) {
434                LOGE(" sqlite3_bind_int error %d", err);
435                free(base16buf);
436                break;
437            }
438        }
439
440        err = sqlite3_step(statement);
441        free(base16buf);
442
443        if (err != SQLITE_DONE) {
444            LOGE(" sqlite3_step error %d", err);
445            break;
446        }
447        numTokens++;
448        if (numTokens == 1) {
449            // first call
450            u_strtok_r(origData, delim, &state);
451        }
452    } while ((token = u_strtok_r(NULL, delim, &state)) != NULL);
453    sqlite3_result_int(context, numTokens);
454}
455
456static void localized_collator_dtor(UCollator* collator)
457{
458    ucol_close(collator);
459}
460
461#define LOCALIZED_COLLATOR_NAME "LOCALIZED"
462
463extern "C" int register_localized_collators(sqlite3* handle, const char* systemLocale, int utf16Storage)
464{
465    int err;
466    UErrorCode status = U_ZERO_ERROR;
467    void* icudata;
468
469    UCollator* collator = ucol_open(systemLocale, &status);
470    if (U_FAILURE(status)) {
471        return -1;
472    }
473
474    ucol_setAttribute(collator, UCOL_STRENGTH, UCOL_PRIMARY, &status);
475    if (U_FAILURE(status)) {
476        return -1;
477    }
478
479    status = U_ZERO_ERROR;
480    char buf[1024];
481    int n = ucol_getShortDefinitionString(collator, NULL, buf, 1024, &status);
482
483    if (utf16Storage) {
484        err = sqlite3_create_collation_v2(handle, LOCALIZED_COLLATOR_NAME, SQLITE_UTF16, collator,
485                collate16, (void(*)(void*))localized_collator_dtor);
486    } else {
487        err = sqlite3_create_collation_v2(handle, LOCALIZED_COLLATOR_NAME, SQLITE_UTF8, collator,
488                collate8, (void(*)(void*))localized_collator_dtor);
489    }
490    if (err != SQLITE_OK) {
491        return err;
492    }
493
494    // Register the _TOKENIZE function
495    err = sqlite3_create_function(handle, "_TOKENIZE", 4, SQLITE_UTF16, collator, tokenize, NULL, NULL);
496    if (err != SQLITE_OK) {
497        return err;
498    }
499    err = sqlite3_create_function(handle, "_TOKENIZE", 5, SQLITE_UTF16, collator, tokenize, NULL, NULL);
500    if (err != SQLITE_OK) {
501        return err;
502    }
503    err = sqlite3_create_function(handle, "_TOKENIZE", 6, SQLITE_UTF16, collator, tokenize, NULL, NULL);
504    if (err != SQLITE_OK) {
505        return err;
506    }
507
508    return SQLITE_OK;
509}
510
511
512extern "C" int register_android_functions(sqlite3 * handle, int utf16Storage)
513{
514    int err;
515    UErrorCode status = U_ZERO_ERROR;
516
517    UCollator * collator = ucol_open(NULL, &status);
518    if (U_FAILURE(status)) {
519        return -1;
520    }
521
522    if (utf16Storage) {
523        // Note that text should be stored as UTF-16
524        err = sqlite3_exec(handle, "PRAGMA encoding = 'UTF-16'", 0, 0, 0);
525        if (err != SQLITE_OK) {
526            return err;
527        }
528
529        // Register the UNICODE collation
530        err = sqlite3_create_collation_v2(handle, "UNICODE", SQLITE_UTF16, collator, collate16,
531                (void(*)(void*))localized_collator_dtor);
532    } else {
533        err = sqlite3_create_collation_v2(handle, "UNICODE", SQLITE_UTF8, collator, collate8,
534                (void(*)(void*))localized_collator_dtor);
535    }
536
537    if (err != SQLITE_OK) {
538        return err;
539    }
540
541    // Register the PHONE_NUM_EQUALS function
542    err = sqlite3_create_function(
543        handle, "PHONE_NUMBERS_EQUAL", 2,
544        SQLITE_UTF8, NULL, phone_numbers_equal, NULL, NULL);
545    if (err != SQLITE_OK) {
546        return err;
547    }
548
549    // Register the PHONE_NUM_EQUALS function with an additional argument "use_strict"
550    err = sqlite3_create_function(
551        handle, "PHONE_NUMBERS_EQUAL", 3,
552        SQLITE_UTF8, NULL, phone_numbers_equal, NULL, NULL);
553    if (err != SQLITE_OK) {
554        return err;
555    }
556
557    // Register the _DELETE_FILE function
558    err = sqlite3_create_function(handle, "_DELETE_FILE", 1, SQLITE_UTF8, NULL, delete_file, NULL, NULL);
559    if (err != SQLITE_OK) {
560        return err;
561    }
562
563#if ENABLE_ANDROID_LOG
564    // Register the _LOG function
565    err = sqlite3_create_function(handle, "_LOG", 1, SQLITE_UTF8, NULL, android_log, NULL, NULL);
566    if (err != SQLITE_OK) {
567        return err;
568    }
569#endif
570
571    // Register the GET_PHONETICALLY_SORTABLE_STRING function
572    err = sqlite3_create_function(handle,
573                                  "GET_PHONETICALLY_SORTABLE_STRING",
574                                  1, SQLITE_UTF8, NULL,
575                                  get_phonetically_sortable_string,
576                                  NULL, NULL);
577    if (err != SQLITE_OK) {
578        return err;
579    }
580
581    // Register the GET_NORMALIZED_STRING function
582    err = sqlite3_create_function(handle,
583                                  "GET_NORMALIZED_STRING",
584                                  1, SQLITE_UTF8, NULL,
585                                  get_normalized_string,
586                                  NULL, NULL);
587    if (err != SQLITE_OK) {
588        return err;
589    }
590
591    // Register the GET_PHONEBOOK_INDEX function
592    err = sqlite3_create_function(handle,
593        "GET_PHONEBOOK_INDEX",
594        2, SQLITE_UTF8, NULL,
595        get_phonebook_index,
596        NULL, NULL);
597    if (err != SQLITE_OK) {
598        return err;
599    }
600
601    return SQLITE_OK;
602}
603