1// Copyright 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "content/browser/speech/speech_recognizer_impl_android.h"
6
7#include "base/android/jni_android.h"
8#include "base/android/jni_array.h"
9#include "base/android/jni_string.h"
10#include "base/android/scoped_java_ref.h"
11#include "base/bind.h"
12#include "base/strings/utf_string_conversions.h"
13#include "content/public/browser/browser_thread.h"
14#include "content/public/browser/speech_recognition_event_listener.h"
15#include "content/public/browser/speech_recognition_manager.h"
16#include "content/public/browser/speech_recognition_session_config.h"
17#include "content/public/common/speech_recognition_grammar.h"
18#include "content/public/common/speech_recognition_result.h"
19#include "jni/SpeechRecognition_jni.h"
20
21using base::android::AppendJavaStringArrayToStringVector;
22using base::android::AttachCurrentThread;
23using base::android::ConvertUTF8ToJavaString;
24using base::android::GetApplicationContext;
25using base::android::JavaFloatArrayToFloatVector;
26
27namespace content {
28
29SpeechRecognizerImplAndroid::SpeechRecognizerImplAndroid(
30    SpeechRecognitionEventListener* listener,
31    int session_id)
32    : SpeechRecognizer(listener, session_id),
33      state_(STATE_IDLE) {
34}
35
36SpeechRecognizerImplAndroid::~SpeechRecognizerImplAndroid() { }
37
38void SpeechRecognizerImplAndroid::StartRecognition(
39    const std::string& device_id) {
40  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
41  // TODO(xians): Open the correct device for speech on Android.
42  BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
43      &SpeechRecognitionEventListener::OnRecognitionStart,
44      base::Unretained(listener()),
45      session_id()));
46  SpeechRecognitionSessionConfig config =
47      SpeechRecognitionManager::GetInstance()->GetSessionConfig(session_id());
48  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
49      &content::SpeechRecognizerImplAndroid::StartRecognitionOnUIThread, this,
50      config.language, config.continuous, config.interim_results));
51}
52
53void SpeechRecognizerImplAndroid::StartRecognitionOnUIThread(
54    std::string language, bool continuous, bool interim_results) {
55  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
56  JNIEnv* env = AttachCurrentThread();
57  j_recognition_.Reset(Java_SpeechRecognition_createSpeechRecognition(env,
58      GetApplicationContext(), reinterpret_cast<intptr_t>(this)));
59  Java_SpeechRecognition_startRecognition(env, j_recognition_.obj(),
60      ConvertUTF8ToJavaString(env, language).obj(), continuous,
61      interim_results);
62}
63
64void SpeechRecognizerImplAndroid::AbortRecognition() {
65  if (BrowserThread::CurrentlyOn(BrowserThread::IO)) {
66    state_ = STATE_IDLE;
67    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
68        &content::SpeechRecognizerImplAndroid::AbortRecognition, this));
69    return;
70  }
71  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
72  JNIEnv* env = AttachCurrentThread();
73  if (!j_recognition_.is_null())
74    Java_SpeechRecognition_abortRecognition(env, j_recognition_.obj());
75}
76
77void SpeechRecognizerImplAndroid::StopAudioCapture() {
78  if (BrowserThread::CurrentlyOn(BrowserThread::IO)) {
79    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
80        &content::SpeechRecognizerImplAndroid::StopAudioCapture, this));
81    return;
82  }
83  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
84  JNIEnv* env = AttachCurrentThread();
85  if (!j_recognition_.is_null())
86    Java_SpeechRecognition_stopRecognition(env, j_recognition_.obj());
87}
88
89bool SpeechRecognizerImplAndroid::IsActive() const {
90  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
91  return state_ != STATE_IDLE;
92}
93
94bool SpeechRecognizerImplAndroid::IsCapturingAudio() const {
95  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
96  return state_ == STATE_CAPTURING_AUDIO;
97}
98
99void SpeechRecognizerImplAndroid::OnAudioStart(JNIEnv* env, jobject obj) {
100  if (BrowserThread::CurrentlyOn(BrowserThread::UI)) {
101    BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
102        &SpeechRecognizerImplAndroid::OnAudioStart, this,
103        static_cast<JNIEnv*>(NULL), static_cast<jobject>(NULL)));
104    return;
105  }
106  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
107  state_ = STATE_CAPTURING_AUDIO;
108  listener()->OnAudioStart(session_id());
109}
110
111void SpeechRecognizerImplAndroid::OnSoundStart(JNIEnv* env, jobject obj) {
112  if (BrowserThread::CurrentlyOn(BrowserThread::UI)) {
113    BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
114        &SpeechRecognizerImplAndroid::OnSoundStart, this,
115        static_cast<JNIEnv*>(NULL), static_cast<jobject>(NULL)));
116    return;
117  }
118  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
119  listener()->OnSoundStart(session_id());
120}
121
122void SpeechRecognizerImplAndroid::OnSoundEnd(JNIEnv* env, jobject obj) {
123  if (BrowserThread::CurrentlyOn(BrowserThread::UI)) {
124    BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
125        &SpeechRecognizerImplAndroid::OnSoundEnd, this,
126        static_cast<JNIEnv*>(NULL), static_cast<jobject>(NULL)));
127    return;
128  }
129  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
130  listener()->OnSoundEnd(session_id());
131}
132
133void SpeechRecognizerImplAndroid::OnAudioEnd(JNIEnv* env, jobject obj) {
134  if (BrowserThread::CurrentlyOn(BrowserThread::UI)) {
135    BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
136        &SpeechRecognizerImplAndroid::OnAudioEnd, this,
137        static_cast<JNIEnv*>(NULL), static_cast<jobject>(NULL)));
138    return;
139  }
140  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
141  if (state_ == STATE_CAPTURING_AUDIO)
142    state_ = STATE_AWAITING_FINAL_RESULT;
143  listener()->OnAudioEnd(session_id());
144}
145
146void SpeechRecognizerImplAndroid::OnRecognitionResults(JNIEnv* env, jobject obj,
147    jobjectArray strings, jfloatArray floats, jboolean provisional) {
148  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
149  std::vector<base::string16> options;
150  AppendJavaStringArrayToStringVector(env, strings, &options);
151  std::vector<float> scores(options.size(), 0.0);
152  if (floats != NULL)
153    JavaFloatArrayToFloatVector(env, floats, &scores);
154  SpeechRecognitionResults results;
155  results.push_back(SpeechRecognitionResult());
156  SpeechRecognitionResult& result = results.back();
157  CHECK_EQ(options.size(), scores.size());
158  for (size_t i = 0; i < options.size(); ++i) {
159    result.hypotheses.push_back(SpeechRecognitionHypothesis(
160        options[i], static_cast<double>(scores[i])));
161  }
162  result.is_provisional = provisional;
163  BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
164      &SpeechRecognizerImplAndroid::OnRecognitionResultsOnIOThread,
165      this, results));
166}
167
168void SpeechRecognizerImplAndroid::OnRecognitionResultsOnIOThread(
169    SpeechRecognitionResults const &results) {
170  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
171  listener()->OnRecognitionResults(session_id(), results);
172}
173
174void SpeechRecognizerImplAndroid::OnRecognitionError(JNIEnv* env,
175    jobject obj, jint error) {
176  if (BrowserThread::CurrentlyOn(BrowserThread::UI)) {
177    BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
178        &SpeechRecognizerImplAndroid::OnRecognitionError, this,
179        static_cast<JNIEnv*>(NULL), static_cast<jobject>(NULL), error));
180    return;
181  }
182  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
183  SpeechRecognitionErrorCode code =
184      static_cast<SpeechRecognitionErrorCode>(error);
185  listener()->OnRecognitionError(session_id(), SpeechRecognitionError(code));
186}
187
188void SpeechRecognizerImplAndroid::OnRecognitionEnd(JNIEnv* env,
189    jobject obj) {
190  if (BrowserThread::CurrentlyOn(BrowserThread::UI)) {
191    BrowserThread::PostTask(BrowserThread::IO, FROM_HERE, base::Bind(
192        &SpeechRecognizerImplAndroid::OnRecognitionEnd, this,
193        static_cast<JNIEnv*>(NULL), static_cast<jobject>(NULL)));
194    return;
195  }
196  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
197  state_ = STATE_IDLE;
198  listener()->OnRecognitionEnd(session_id());
199}
200
201// static
202bool SpeechRecognizerImplAndroid::RegisterSpeechRecognizer(JNIEnv* env) {
203  return RegisterNativesImpl(env);
204}
205
206}  // namespace content
207