android_media_MediaSync.cpp revision 05ebffe6bf6cdacc6de4b3bbf480c31869a81661 
1/*
2 * Copyright 2015, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "MediaSync-JNI"
19#include <utils/Log.h>
20
21#include "android_media_MediaSync.h"
22
23#include "android_media_AudioTrack.h"
24#include "android_media_PlaybackSettings.h"
25#include "android_media_SyncSettings.h"
26#include "android_runtime/AndroidRuntime.h"
27#include "android_runtime/android_view_Surface.h"
28#include "jni.h"
29#include "JNIHelp.h"
30
31#include <gui/Surface.h>
32
33#include <media/AudioResamplerPublic.h>
34#include <media/AudioTrack.h>
35#include <media/stagefright/MediaClock.h>
36#include <media/stagefright/MediaSync.h>
37#include <media/stagefright/foundation/ADebug.h>
38#include <media/stagefright/foundation/AString.h>
39
40#include <nativehelper/ScopedLocalRef.h>
41
42namespace android {
43
44struct fields_t {
45    jfieldID context;
46    jfieldID mediaTimestampMediaTimeUsID;
47    jfieldID mediaTimestampNanoTimeID;
48    jfieldID mediaTimestampClockRateID;
49};
50
51static fields_t gFields;
52static PlaybackSettings::fields_t gPlaybackSettingsFields;
53static SyncSettings::fields_t gSyncSettingsFields;
54
55////////////////////////////////////////////////////////////////////////////////
56
57JMediaSync::JMediaSync() {
58    mSync = MediaSync::create();
59}
60
61JMediaSync::~JMediaSync() {
62}
63
64status_t JMediaSync::configureSurface(const sp<IGraphicBufferProducer> &bufferProducer) {
65    return mSync->configureSurface(bufferProducer);
66}
67
68status_t JMediaSync::configureAudioTrack(const sp<AudioTrack> &audioTrack) {
69    return mSync->configureAudioTrack(audioTrack);
70}
71
72status_t JMediaSync::createInputSurface(
73        sp<IGraphicBufferProducer>* bufferProducer) {
74    return mSync->createInputSurface(bufferProducer);
75}
76
77sp<const MediaClock> JMediaSync::getMediaClock() {
78    return mSync->getMediaClock();
79}
80
81status_t JMediaSync::setPlaybackSettings(const AudioPlaybackRate& rate) {
82    return mSync->setPlaybackSettings(rate);
83}
84
85void JMediaSync::getPlaybackSettings(AudioPlaybackRate* rate /* nonnull */) {
86    mSync->getPlaybackSettings(rate);
87}
88
89status_t JMediaSync::setSyncSettings(const AVSyncSettings& syncSettings) {
90    return mSync->setSyncSettings(syncSettings);
91}
92
93void JMediaSync::getSyncSettings(AVSyncSettings* syncSettings /* nonnull */) {
94    mSync->getSyncSettings(syncSettings);
95}
96
97status_t JMediaSync::setVideoFrameRateHint(float rate) {
98    return mSync->setVideoFrameRateHint(rate);
99}
100
101float JMediaSync::getVideoFrameRate() {
102    return mSync->getVideoFrameRate();
103}
104
105status_t JMediaSync::updateQueuedAudioData(
106        int sizeInBytes, int64_t presentationTimeUs) {
107    return mSync->updateQueuedAudioData(sizeInBytes, presentationTimeUs);
108}
109
110status_t JMediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) {
111    return mSync->getPlayTimeForPendingAudioFrames(outTimeUs);
112}
113
114}  // namespace android
115
116////////////////////////////////////////////////////////////////////////////////
117
118using namespace android;
119
120static sp<JMediaSync> setMediaSync(JNIEnv *env, jobject thiz, const sp<JMediaSync> &sync) {
121    sp<JMediaSync> old = (JMediaSync *)env->GetLongField(thiz, gFields.context);
122    if (sync != NULL) {
123        sync->incStrong(thiz);
124    }
125    if (old != NULL) {
126        old->decStrong(thiz);
127    }
128
129    env->SetLongField(thiz, gFields.context, (jlong)sync.get());
130
131    return old;
132}
133
134static sp<JMediaSync> getMediaSync(JNIEnv *env, jobject thiz) {
135    return (JMediaSync *)env->GetLongField(thiz, gFields.context);
136}
137
138static void android_media_MediaSync_release(JNIEnv *env, jobject thiz) {
139    setMediaSync(env, thiz, NULL);
140}
141
142static void throwExceptionAsNecessary(
143        JNIEnv *env, status_t err, const char *msg = NULL) {
144    switch (err) {
145        case NO_ERROR:
146            break;
147
148        case BAD_VALUE:
149            jniThrowException(env, "java/lang/IllegalArgumentException", msg);
150            break;
151
152        case NO_INIT:
153        case INVALID_OPERATION:
154        default:
155            if (err > 0) {
156                break;
157            }
158            AString msgWithErrorCode(msg);
159            msgWithErrorCode.append(" error:");
160            msgWithErrorCode.append(err);
161            jniThrowException(env, "java/lang/IllegalStateException", msgWithErrorCode.c_str());
162            break;
163    }
164}
165
166static void android_media_MediaSync_native_configureSurface(
167        JNIEnv *env, jobject thiz, jobject jsurface) {
168    ALOGV("android_media_MediaSync_configureSurface");
169
170    sp<JMediaSync> sync = getMediaSync(env, thiz);
171    if (sync == NULL) {
172        throwExceptionAsNecessary(env, INVALID_OPERATION);
173        return;
174    }
175
176    sp<IGraphicBufferProducer> bufferProducer;
177    if (jsurface != NULL) {
178        sp<Surface> surface(android_view_Surface_getSurface(env, jsurface));
179        if (surface != NULL) {
180            bufferProducer = surface->getIGraphicBufferProducer();
181        } else {
182            throwExceptionAsNecessary(env, BAD_VALUE, "The surface has been released");
183            return;
184        }
185    }
186
187    status_t err = sync->configureSurface(bufferProducer);
188
189    if (err == INVALID_OPERATION) {
190        throwExceptionAsNecessary(
191                env, INVALID_OPERATION, "Surface has already been configured");
192    } if (err != NO_ERROR) {
193        AString msg("Failed to connect to surface with error ");
194        msg.append(err);
195        throwExceptionAsNecessary(env, BAD_VALUE, msg.c_str());
196    }
197}
198
199static void android_media_MediaSync_native_configureAudioTrack(
200        JNIEnv *env, jobject thiz, jobject jaudioTrack) {
201    ALOGV("android_media_MediaSync_configureAudioTrack");
202
203    sp<JMediaSync> sync = getMediaSync(env, thiz);
204    if (sync == NULL) {
205        throwExceptionAsNecessary(env, INVALID_OPERATION);
206        return;
207    }
208
209    sp<AudioTrack> audioTrack;
210    if (jaudioTrack != NULL) {
211        audioTrack = android_media_AudioTrack_getAudioTrack(env, jaudioTrack);
212        if (audioTrack == NULL) {
213            throwExceptionAsNecessary(env, BAD_VALUE, "The audio track has been released");
214            return;
215        }
216    }
217
218    status_t err = sync->configureAudioTrack(audioTrack);
219
220    if (err == INVALID_OPERATION) {
221        throwExceptionAsNecessary(
222                env, INVALID_OPERATION, "Audio track has already been configured");
223    } if (err != NO_ERROR) {
224        AString msg("Failed to configure audio track with error ");
225        msg.append(err);
226        throwExceptionAsNecessary(env, BAD_VALUE, msg.c_str());
227    }
228}
229
230static jobject android_media_MediaSync_createInputSurface(
231        JNIEnv* env, jobject thiz) {
232    ALOGV("android_media_MediaSync_createInputSurface");
233
234    sp<JMediaSync> sync = getMediaSync(env, thiz);
235    if (sync == NULL) {
236        throwExceptionAsNecessary(env, INVALID_OPERATION);
237        return NULL;
238    }
239
240    // Tell the MediaSync that we want to use a Surface as input.
241    sp<IGraphicBufferProducer> bufferProducer;
242    status_t err = sync->createInputSurface(&bufferProducer);
243    if (err != NO_ERROR) {
244        throwExceptionAsNecessary(env, INVALID_OPERATION);
245        return NULL;
246    }
247
248    // Wrap the IGBP in a Java-language Surface.
249    return android_view_Surface_createFromIGraphicBufferProducer(env,
250            bufferProducer);
251}
252
253static void android_media_MediaSync_native_updateQueuedAudioData(
254        JNIEnv *env, jobject thiz, jint sizeInBytes, jlong presentationTimeUs) {
255    sp<JMediaSync> sync = getMediaSync(env, thiz);
256    if (sync == NULL) {
257        throwExceptionAsNecessary(env, INVALID_OPERATION);
258        return;
259    }
260
261    status_t err = sync->updateQueuedAudioData(sizeInBytes, presentationTimeUs);
262    if (err != NO_ERROR) {
263        throwExceptionAsNecessary(env, err);
264        return;
265    }
266}
267
268static jboolean android_media_MediaSync_native_getTimestamp(
269        JNIEnv *env, jobject thiz, jobject timestamp) {
270    sp<JMediaSync> sync = getMediaSync(env, thiz);
271    if (sync == NULL) {
272        throwExceptionAsNecessary(env, INVALID_OPERATION);
273        return JNI_FALSE;
274    }
275
276    sp<const MediaClock> mediaClock = sync->getMediaClock();
277    if (mediaClock == NULL) {
278        return JNI_FALSE;
279    }
280
281    int64_t nowUs = ALooper::GetNowUs();
282    int64_t mediaUs = 0;
283    if (mediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
284        return JNI_FALSE;
285    }
286
287    env->SetLongField(timestamp, gFields.mediaTimestampMediaTimeUsID,
288            (jlong)mediaUs);
289    env->SetLongField(timestamp, gFields.mediaTimestampNanoTimeID,
290            (jlong)(nowUs * 1000));
291    env->SetFloatField(timestamp, gFields.mediaTimestampClockRateID,
292            (jfloat)mediaClock->getPlaybackRate());
293    return JNI_TRUE;
294}
295
296static jlong android_media_MediaSync_native_getPlayTimeForPendingAudioFrames(
297        JNIEnv *env, jobject thiz) {
298    sp<JMediaSync> sync = getMediaSync(env, thiz);
299    if (sync == NULL) {
300        throwExceptionAsNecessary(env, INVALID_OPERATION);
301    }
302
303    int64_t playTimeUs = 0;
304    status_t err = sync->getPlayTimeForPendingAudioFrames(&playTimeUs);
305    if (err != NO_ERROR) {
306        throwExceptionAsNecessary(env, err);
307    }
308    return (jlong)playTimeUs;
309}
310
311static jfloat android_media_MediaSync_setPlaybackSettings(
312        JNIEnv *env, jobject thiz, jobject settings) {
313    sp<JMediaSync> sync = getMediaSync(env, thiz);
314    if (sync == NULL) {
315        throwExceptionAsNecessary(env, INVALID_OPERATION);
316        return (jfloat)0.f;
317    }
318
319    PlaybackSettings pbs;
320    pbs.fillFromJobject(env, gPlaybackSettingsFields, settings);
321    ALOGV("setPlaybackSettings: %d:%f %d:%f %d:%u %d:%u",
322            pbs.speedSet, pbs.audioRate.mSpeed,
323            pbs.pitchSet, pbs.audioRate.mPitch,
324            pbs.audioFallbackModeSet, pbs.audioRate.mFallbackMode,
325            pbs.audioStretchModeSet, pbs.audioRate.mStretchMode);
326
327    AudioPlaybackRate rate;
328    sync->getPlaybackSettings(&rate);
329    bool updatedRate = false;
330    if (pbs.speedSet) {
331        rate.mSpeed = pbs.audioRate.mSpeed;
332        updatedRate = true;
333    }
334    if (pbs.pitchSet) {
335        rate.mPitch = pbs.audioRate.mPitch;
336        updatedRate = true;
337    }
338    if (pbs.audioFallbackModeSet) {
339        rate.mFallbackMode = pbs.audioRate.mFallbackMode;
340        updatedRate = true;
341    }
342    if (pbs.audioStretchModeSet) {
343        rate.mStretchMode = pbs.audioRate.mStretchMode;
344        updatedRate = true;
345    }
346    if (updatedRate) {
347        status_t err = sync->setPlaybackSettings(rate);
348        if (err != OK) {
349            throwExceptionAsNecessary(env, err);
350            return (jfloat)0.f;
351        }
352    }
353
354    sp<const MediaClock> mediaClock = sync->getMediaClock();
355    if (mediaClock == NULL) {
356        return (jfloat)0.f;
357    }
358
359    return (jfloat)mediaClock->getPlaybackRate();
360}
361
362static jobject android_media_MediaSync_getPlaybackSettings(
363        JNIEnv *env, jobject thiz) {
364    sp<JMediaSync> sync = getMediaSync(env, thiz);
365    if (sync == NULL) {
366        throwExceptionAsNecessary(env, INVALID_OPERATION);
367        return NULL;
368    }
369
370    PlaybackSettings pbs;
371    AudioPlaybackRate &audioRate = pbs.audioRate;
372    sync->getPlaybackSettings(&audioRate);
373    ALOGV("getPlaybackSettings: %f %f %d %d",
374            audioRate.mSpeed, audioRate.mPitch, audioRate.mFallbackMode, audioRate.mStretchMode);
375
376    pbs.speedSet = true;
377    pbs.pitchSet = true;
378    pbs.audioFallbackModeSet = true;
379    pbs.audioStretchModeSet = true;
380
381    return pbs.asJobject(env, gPlaybackSettingsFields);
382}
383
384static jfloat android_media_MediaSync_setSyncSettings(
385        JNIEnv *env, jobject thiz, jobject settings) {
386    sp<JMediaSync> sync = getMediaSync(env, thiz);
387    if (sync == NULL) {
388        throwExceptionAsNecessary(env, INVALID_OPERATION);
389        return (jfloat)0.f;
390    }
391
392    SyncSettings scs;
393    scs.fillFromJobject(env, gSyncSettingsFields, settings);
394    ALOGV("setSyncSettings: %d:%d %d:%d %d:%f %d:%f",
395            scs.syncSourceSet, scs.sync.mSource,
396            scs.audioAdjustModeSet, scs.sync.mAudioAdjustMode,
397            scs.toleranceSet, scs.sync.mTolerance,
398            scs.frameRateSet, scs.frameRate);
399
400    AVSyncSettings avsync;
401    sync->getSyncSettings(&avsync);
402    bool updatedSync = false;
403    status_t err = OK;
404    if (scs.syncSourceSet) {
405        avsync.mSource = scs.sync.mSource;
406        updatedSync = true;
407    }
408    if (scs.audioAdjustModeSet) {
409        avsync.mAudioAdjustMode = scs.sync.mAudioAdjustMode;
410        updatedSync = true;
411    }
412    if (scs.toleranceSet) {
413        avsync.mTolerance = scs.sync.mTolerance;
414        updatedSync = true;
415    }
416    if (updatedSync) {
417        err = sync->setSyncSettings(avsync);
418    }
419
420    if (scs.frameRateSet && err == OK) {
421        err = sync->setVideoFrameRateHint(scs.frameRate);
422    }
423    if (err != OK) {
424        throwExceptionAsNecessary(env, err);
425        return (jfloat)0.f;
426    }
427
428    sp<const MediaClock> mediaClock = sync->getMediaClock();
429    if (mediaClock == NULL) {
430        return (jfloat)0.f;
431    }
432
433    return (jfloat)mediaClock->getPlaybackRate();
434}
435
436static jobject android_media_MediaSync_getSyncSettings(JNIEnv *env, jobject thiz) {
437    sp<JMediaSync> sync = getMediaSync(env, thiz);
438    if (sync == NULL) {
439        throwExceptionAsNecessary(env, INVALID_OPERATION);
440        return NULL;
441    }
442
443    SyncSettings scs;
444    sync->getSyncSettings(&scs.sync);
445    scs.frameRate = sync->getVideoFrameRate();
446
447    ALOGV("getSyncSettings: %d %d %f %f",
448            scs.sync.mSource, scs.sync.mAudioAdjustMode, scs.sync.mTolerance, scs.frameRate);
449
450    // sanity check settings
451    if (scs.sync.mSource >= AVSYNC_SOURCE_MAX
452            || scs.sync.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
453            || scs.sync.mTolerance < 0.f
454            || scs.sync.mTolerance >= AVSYNC_TOLERANCE_MAX) {
455        throwExceptionAsNecessary(env, INVALID_OPERATION);
456        return NULL;
457    }
458
459    scs.syncSourceSet = true;
460    scs.audioAdjustModeSet = true;
461    scs.toleranceSet = true;
462    scs.frameRateSet = scs.frameRate >= 0.f;
463
464    return scs.asJobject(env, gSyncSettingsFields);
465}
466
467static void android_media_MediaSync_native_init(JNIEnv *env) {
468    ScopedLocalRef<jclass> clazz(env, env->FindClass("android/media/MediaSync"));
469    CHECK(clazz.get() != NULL);
470
471    gFields.context = env->GetFieldID(clazz.get(), "mNativeContext", "J");
472    CHECK(gFields.context != NULL);
473
474    clazz.reset(env->FindClass("android/media/MediaTimestamp"));
475    CHECK(clazz.get() != NULL);
476
477    gFields.mediaTimestampMediaTimeUsID =
478        env->GetFieldID(clazz.get(), "mediaTimeUs", "J");
479    CHECK(gFields.mediaTimestampMediaTimeUsID != NULL);
480
481    gFields.mediaTimestampNanoTimeID =
482        env->GetFieldID(clazz.get(), "nanoTime", "J");
483    CHECK(gFields.mediaTimestampNanoTimeID != NULL);
484
485    gFields.mediaTimestampClockRateID =
486        env->GetFieldID(clazz.get(), "clockRate", "F");
487    CHECK(gFields.mediaTimestampClockRateID != NULL);
488
489    gSyncSettingsFields.init(env);
490    gPlaybackSettingsFields.init(env);
491}
492
493static void android_media_MediaSync_native_setup(JNIEnv *env, jobject thiz) {
494    sp<JMediaSync> sync = new JMediaSync();
495
496    setMediaSync(env, thiz, sync);
497}
498
499static void android_media_MediaSync_native_finalize(JNIEnv *env, jobject thiz) {
500    android_media_MediaSync_release(env, thiz);
501}
502
503static JNINativeMethod gMethods[] = {
504    { "native_configureSurface",
505      "(Landroid/view/Surface;)V",
506      (void *)android_media_MediaSync_native_configureSurface },
507
508    { "native_configureAudioTrack",
509      "(Landroid/media/AudioTrack;I)V",
510      (void *)android_media_MediaSync_native_configureAudioTrack },
511
512    { "createInputSurface", "()Landroid/view/Surface;",
513      (void *)android_media_MediaSync_createInputSurface },
514
515    { "native_updateQueuedAudioData",
516      "(IJ)V",
517      (void *)android_media_MediaSync_native_updateQueuedAudioData },
518
519    { "native_getTimestamp",
520      "(Landroid/media/MediaTimestamp;)Z",
521      (void *)android_media_MediaSync_native_getTimestamp },
522
523    { "native_getPlayTimeForPendingAudioFrames",
524      "()J",
525      (void *)android_media_MediaSync_native_getPlayTimeForPendingAudioFrames },
526
527    { "native_init", "()V", (void *)android_media_MediaSync_native_init },
528
529    { "native_setup", "()V", (void *)android_media_MediaSync_native_setup },
530
531    { "native_release", "()V", (void *)android_media_MediaSync_release },
532
533    { "native_setPlaybackSettings", "(Landroid/media/PlaybackSettings;)F",
534      (void *)android_media_MediaSync_setPlaybackSettings },
535
536    { "getPlaybackSettings", "()Landroid/media/PlaybackSettings;",
537      (void *)android_media_MediaSync_getPlaybackSettings },
538
539    { "native_setSyncSettings", "(Landroid/media/SyncSettings;)F",
540      (void *)android_media_MediaSync_setSyncSettings },
541
542    { "getSyncSettings", "()Landroid/media/SyncSettings;",
543      (void *)android_media_MediaSync_getSyncSettings },
544
545    { "native_finalize", "()V", (void *)android_media_MediaSync_native_finalize },
546};
547
548int register_android_media_MediaSync(JNIEnv *env) {
549    return AndroidRuntime::registerNativeMethods(
550                   env, "android/media/MediaSync", gMethods, NELEM(gMethods));
551}
552