1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package org.chromium.latency.walt;
18
19import android.content.BroadcastReceiver;
20import android.content.Context;
21import android.content.Intent;
22import android.graphics.Color;
23import android.os.Bundle;
24import android.os.Handler;
25import android.support.v4.app.Fragment;
26import android.support.v7.app.ActionBar;
27import android.support.v7.app.AlertDialog;
28import android.support.v7.app.AppCompatActivity;
29import android.text.method.ScrollingMovementMethod;
30import android.view.Choreographer;
31import android.view.LayoutInflater;
32import android.view.View;
33import android.view.ViewGroup;
34import android.view.WindowManager;
35import android.widget.ArrayAdapter;
36import android.widget.Spinner;
37import android.widget.TextView;
38
39import com.github.mikephil.charting.charts.LineChart;
40import com.github.mikephil.charting.components.Description;
41import com.github.mikephil.charting.data.Entry;
42import com.github.mikephil.charting.data.LineData;
43import com.github.mikephil.charting.data.LineDataSet;
44
45import java.io.IOException;
46import java.util.ArrayList;
47import java.util.List;
48import java.util.Locale;
49
50import static org.chromium.latency.walt.Utils.getBooleanPreference;
51import static org.chromium.latency.walt.Utils.getIntPreference;
52
53/**
54 * Measurement of screen response time when switching between black and white.
55 */
56public class ScreenResponseFragment extends Fragment
57        implements View.OnClickListener, RobotAutomationListener {
58
59    private static final int CURVE_TIMEOUT = 1000;  // milliseconds
60    private static final int CURVE_BLINK_TIME = 250;  // milliseconds
61    private static final int W2B_INDEX = 0;
62    private static final int B2W_INDEX = 1;
63    private SimpleLogger logger;
64    private TraceLogger traceLogger = null;
65    private WaltDevice waltDevice;
66    private Handler handler = new Handler();
67    private TextView blackBox;
68    private View startButton;
69    private View stopButton;
70    private Spinner spinner;
71    private LineChart brightnessChart;
72    private HistogramChart latencyChart;
73    private View brightnessChartLayout;
74    private View buttonBarView;
75    private FastPathSurfaceView fastSurfaceView;
76    private int timesToBlink;
77    private boolean shouldShowLatencyChart = false;
78    private boolean isTestRunning = false;
79    private boolean enableFullScreen = false;
80    private boolean isFastPathGraphics = false;
81    int initiatedBlinks = 0;
82    int detectedBlinks = 0;
83    boolean isBoxWhite = false;
84    long lastFrameStartTime;
85    long lastFrameCallbackTime;
86    long lastSetBackgroundTime;
87    ArrayList<Double> deltas_w2b = new ArrayList<>();
88    ArrayList<Double> deltas_b2w = new ArrayList<>();
89    ArrayList<Double> deltas = new ArrayList<>();
90    private static final int color_gray = Color.argb(0xFF, 0xBB, 0xBB, 0xBB);
91    private StringBuilder brightnessCurveData;
92
93    private BroadcastReceiver logReceiver = new BroadcastReceiver() {
94        @Override
95        public void onReceive(Context context, Intent intent) {
96            if (!isTestRunning) {
97                String msg = intent.getStringExtra("message");
98                blackBox.append(msg + "\n");
99            }
100        }
101    };
102
103    public ScreenResponseFragment() {
104        // Required empty public constructor
105    }
106
107    @Override
108    public View onCreateView(LayoutInflater inflater, ViewGroup container,
109                             Bundle savedInstanceState) {
110        timesToBlink = getIntPreference(getContext(), R.string.preference_screen_blinks, 20);
111        shouldShowLatencyChart = getBooleanPreference(getContext(), R.string.preference_show_blink_histogram, true);
112        enableFullScreen = getBooleanPreference(getContext(), R.string.preference_screen_fullscreen, true);
113        if (getBooleanPreference(getContext(), R.string.preference_systrace, true)) {
114            traceLogger = TraceLogger.getInstance();
115        }
116        waltDevice = WaltDevice.getInstance(getContext());
117        logger = SimpleLogger.getInstance(getContext());
118
119        // Inflate the layout for this fragment
120        final View view = inflater.inflate(R.layout.fragment_screen_response, container, false);
121        stopButton = view.findViewById(R.id.button_stop_screen_response);
122        startButton = view.findViewById(R.id.button_start_screen_response);
123        blackBox = (TextView) view.findViewById(R.id.txt_black_box_screen);
124        fastSurfaceView = (FastPathSurfaceView) view.findViewById(R.id.fast_path_surface);
125        spinner = (Spinner) view.findViewById(R.id.spinner_screen_response);
126        buttonBarView = view.findViewById(R.id.button_bar);
127        ArrayAdapter<CharSequence> modeAdapter = ArrayAdapter.createFromResource(getContext(),
128                R.array.screen_response_mode_array, android.R.layout.simple_spinner_item);
129        modeAdapter.setDropDownViewResource(R.layout.support_simple_spinner_dropdown_item);
130        spinner.setAdapter(modeAdapter);
131        stopButton.setEnabled(false);
132        blackBox.setMovementMethod(new ScrollingMovementMethod());
133        brightnessChartLayout = view.findViewById(R.id.brightness_chart_layout);
134        view.findViewById(R.id.button_close_chart).setOnClickListener(this);
135        brightnessChart = (LineChart) view.findViewById(R.id.chart);
136        latencyChart = (HistogramChart) view.findViewById(R.id.latency_chart);
137
138        if (getBooleanPreference(getContext(), R.string.preference_auto_increase_brightness, true)) {
139            increaseScreenBrightness();
140        }
141        return view;
142    }
143
144    @Override
145    public void onResume() {
146        super.onResume();
147        logger.registerReceiver(logReceiver);
148        // Register this fragment class as the listener for some button clicks
149        startButton.setOnClickListener(this);
150        stopButton.setOnClickListener(this);
151    }
152
153    @Override
154    public void onPause() {
155        logger.unregisterReceiver(logReceiver);
156        super.onPause();
157    }
158
159    void startBlinkLatency() {
160        setFullScreen(enableFullScreen);
161        deltas.clear();
162        deltas_b2w.clear();
163        deltas_w2b.clear();
164        if (shouldShowLatencyChart) {
165            latencyChart.clearData();
166            latencyChart.setVisibility(View.VISIBLE);
167            latencyChart.setLabel(W2B_INDEX, "White-to-black");
168            latencyChart.setLabel(B2W_INDEX, "Black-to-white");
169        }
170        initiatedBlinks = 0;
171        detectedBlinks = 0;
172        if (isFastPathGraphics) {
173            blackBox.setVisibility(View.GONE);
174            fastSurfaceView.setVisibility(View.VISIBLE);
175            fastSurfaceView.setRectColor(Color.WHITE);
176        } else {
177            blackBox.setText("");
178            blackBox.setBackgroundColor(Color.WHITE);
179        }
180        isBoxWhite = true;
181
182        handler.postDelayed(startBlinking, enableFullScreen ? 800 : 300);
183    }
184
185    Runnable startBlinking = new Runnable() {
186        @Override
187        public void run() {
188            try {
189                // Check for PWM
190                WaltDevice.TriggerMessage tmsg = waltDevice.readTriggerMessage(WaltDevice.CMD_SEND_LAST_SCREEN);
191                logger.log("Blink count was: " + tmsg.count);
192
193                waltDevice.softReset();
194                waltDevice.syncClock(); // Note, sync also sends CMD_RESET (but not simpleSync).
195                waltDevice.command(WaltDevice.CMD_AUTO_SCREEN_ON);
196                waltDevice.startListener();
197            } catch (IOException e) {
198                logger.log("Error: " + e.getMessage());
199            }
200
201            // Register a callback for triggers
202            waltDevice.setTriggerHandler(triggerHandler);
203
204            // post doBlink runnable
205            handler.postDelayed(doBlinkRunnable, 100);
206        }
207    };
208
209    Runnable doBlinkRunnable = new Runnable() {
210        @Override
211        public void run() {
212            if (!isTestRunning) return;
213            logger.log("======\ndoBlink.run(), initiatedBlinks = " + initiatedBlinks + " detectedBlinks = " + detectedBlinks);
214            // Check if we saw some transitions without blinking, this would usually mean
215            // the screen has PWM enabled, warn and ask the user to turn it off.
216            if (initiatedBlinks == 0 && detectedBlinks > 1) {
217                logger.log("Unexpected blinks detected, probably PWM, turn it off");
218                isTestRunning = false;
219                stopButton.setEnabled(false);
220                startButton.setEnabled(true);
221                showPwmDialog();
222                return;
223            }
224
225            if (initiatedBlinks >= timesToBlink) {
226                isTestRunning = false;
227                finishAndShowStats();
228                return;
229            }
230
231            // * 2 flip the screen, save time as last flip time (last flip direction?)
232
233            isBoxWhite = !isBoxWhite;
234            int nextColor = isBoxWhite ? Color.WHITE : Color.BLACK;
235            initiatedBlinks++;
236            if (traceLogger != null) {
237                traceLogger.log(RemoteClockInfo.microTime(), RemoteClockInfo.microTime() + 1000,
238                        "Request-to-" + (isBoxWhite ? "white" : "black"),
239                        "Application has called setBackgroundColor at start of bar");
240            }
241            if (isFastPathGraphics) {
242                fastSurfaceView.setRectColor(nextColor);
243            } else {
244                blackBox.setBackgroundColor(nextColor);
245            }
246            lastSetBackgroundTime = waltDevice.clock.micros();
247
248            // Set up a callback to run on next frame render to collect the timestamp
249            Choreographer.getInstance().postFrameCallback(new Choreographer.FrameCallback() {
250                @Override
251                public void doFrame(long frameTimeNanos) {
252                    // frameTimeNanos is the time in nanoseconds when the frame started being
253                    // rendered, in the nanoTime() timebase.
254                    lastFrameStartTime = frameTimeNanos / 1000 - waltDevice.clock.baseTime;
255                    lastFrameCallbackTime = System.nanoTime() / 1000 - waltDevice.clock.baseTime;
256                }
257            });
258
259            // Repost doBlink to some far away time to blink again even if nothing arrives from
260            // Teensy. This callback will almost always get cancelled by onIncomingTimestamp()
261            handler.postDelayed(doBlinkRunnable, 550 + (long) (Math.random()*100));
262        }
263    };
264
265    private void showPwmDialog() {
266        AlertDialog.Builder builder = new AlertDialog.Builder(getActivity());
267        builder.setMessage("Detected extra blinks, please set your brightness to max")
268                .setTitle("Unexpected Blinks")
269                .setPositiveButton("OK", null);
270        AlertDialog dialog = builder.create();
271        dialog.show();
272    }
273
274    private WaltDevice.TriggerHandler triggerHandler = new WaltDevice.TriggerHandler() {
275        @Override
276        public void onReceive(WaltDevice.TriggerMessage tmsg) {
277            // Remove the far away doBlink callback
278            handler.removeCallbacks(doBlinkRunnable);
279
280            detectedBlinks++;
281            logger.log("blink counts " + initiatedBlinks + " " + detectedBlinks);
282            if (initiatedBlinks == 0) {
283                if (detectedBlinks < 5) {
284                    logger.log("got incoming but initiatedBlinks = 0");
285                    return;
286                } else {
287                    logger.log("Looks like PWM is used for this screen, turn auto brightness off and set it to max brightness");
288                    showPwmDialog();
289                    return;
290                }
291            }
292
293            if (traceLogger != null) {
294                final long startTimeMicros = lastFrameStartTime + waltDevice.clock.baseTime;
295                final long finishTimeMicros = tmsg.t + waltDevice.clock.baseTime;
296                traceLogger.log(startTimeMicros, finishTimeMicros,
297                        isBoxWhite ? "Black-to-white" : "White-to-black",
298                        "Bar starts at beginning of frame and ends when photosensor detects blink");
299            }
300
301            double dt = (tmsg.t - lastFrameStartTime) / 1000.;
302            deltas.add(dt);
303            if (isBoxWhite) {  // Current color is the color we transitioned to
304                deltas_b2w.add(dt);
305            } else {
306                deltas_w2b.add(dt);
307            }
308            if (shouldShowLatencyChart) latencyChart.addEntry(isBoxWhite ? B2W_INDEX : W2B_INDEX, dt);
309
310            // Other times can be important, logging them to allow more detailed analysis
311            logger.log(String.format(Locale.US,
312                    "Times [ms]: setBG:%.3f callback:%.3f physical:%.3f black2white:%d",
313                    (lastSetBackgroundTime - lastFrameStartTime) / 1000.0,
314                    (lastFrameCallbackTime - lastFrameStartTime) / 1000.0,
315                    dt,
316                    isBoxWhite ? 1 : 0
317            ));
318            if (traceLogger != null) {
319                traceLogger.log(lastFrameCallbackTime + waltDevice.clock.baseTime,
320                        lastFrameCallbackTime + waltDevice.clock.baseTime + 1000,
321                        isBoxWhite ? "FrameCallback Black-to-white" : "FrameCallback White-to-black",
322                        "FrameCallback was called at start of bar");
323            }
324            // Schedule another blink soon-ish
325            handler.postDelayed(doBlinkRunnable, 40 + (long) (Math.random()*20));
326        }
327    };
328
329
330    void finishAndShowStats() {
331        setFullScreen(false);
332        // Stop the USB listener
333        waltDevice.stopListener();
334
335        // Unregister trigger handler
336        waltDevice.clearTriggerHandler();
337
338        waltDevice.sendAndFlush(WaltDevice.CMD_AUTO_SCREEN_OFF);
339
340        waltDevice.checkDrift();
341
342        // Show deltas and the median
343        /* // Debug printouts
344        logger.log("deltas = array(" + deltas.toString() + ")");
345        logger.log("deltas_w2b = array(" + deltas_w2b.toString() + ")");
346        logger.log("deltas_b2w = array(" + deltas_b2w.toString() + ")");
347        */
348
349        double median_b2w = Utils.median(deltas_b2w);
350        double median_w2b = Utils.median(deltas_w2b);
351        logger.log(String.format(Locale.US,
352                "\n-------------------------------\n" +
353                        "Median screen response latencies (N=%d):\n" +
354                        "Black to white: %.1f ms (N=%d)\n" +
355                        "White to black: %.1f ms (N=%d)\n" +
356                        "Average: %.1f ms\n" +
357                        "-------------------------------\n",
358                deltas.size(),
359                median_b2w, deltas_b2w.size(),
360                median_w2b, deltas_w2b.size(),
361                (median_b2w + median_w2b) / 2
362        ));
363
364        if (traceLogger != null) traceLogger.flush(getContext());
365        fastSurfaceView.setVisibility(View.GONE);
366        blackBox.setVisibility(View.VISIBLE);
367        blackBox.setText(logger.getLogText());
368        blackBox.setMovementMethod(new ScrollingMovementMethod());
369        blackBox.setBackgroundColor(color_gray);
370        stopButton.setEnabled(false);
371        startButton.setEnabled(true);
372        if (shouldShowLatencyChart) {
373            latencyChart.setLabel(W2B_INDEX, String.format(Locale.US, "White-to-black m=%.1f ms", median_w2b));
374            latencyChart.setLabel(B2W_INDEX, String.format(Locale.US, "Black-to-white m=%.1f ms", median_b2w));
375        }
376        LogUploader.uploadIfAutoEnabled(getContext());
377    }
378
379    @Override
380    public void onClick(View v) {
381        if (v.getId() == R.id.button_stop_screen_response) {
382            isTestRunning = false;
383            handler.removeCallbacks(doBlinkRunnable);
384            handler.removeCallbacks(startBlinking);
385            finishAndShowStats();
386            return;
387        }
388
389        if (v.getId() == R.id.button_start_screen_response) {
390            brightnessChartLayout.setVisibility(View.GONE);
391            latencyChart.setVisibility(View.GONE);
392            if (!waltDevice.isConnected()) {
393                logger.log("Error starting test: WALT is not connected");
394                return;
395            }
396
397            isTestRunning = true;
398            startButton.setEnabled(false);
399            blackBox.setBackgroundColor(Color.BLACK);
400            blackBox.setText("");
401            isFastPathGraphics = false;
402            final int spinnerPosition = spinner.getSelectedItemPosition();
403            if (spinnerPosition == 0) {
404                logger.log("Starting screen response measurement");
405                stopButton.setEnabled(true);
406                startBlinkLatency();
407            } else if (spinnerPosition == 1) {
408                logger.log("Starting screen brightness curve measurement");
409                startBrightnessCurve();
410            } else if (spinnerPosition == 2) {
411                logger.log("Starting fast-path screen response measurement");
412                isFastPathGraphics = true;
413                startBlinkLatency();
414            } else {
415                logger.log("ERROR: Spinner position is out of range");
416            }
417            return;
418        }
419
420        if (v.getId() == R.id.button_close_chart) {
421            brightnessChartLayout.setVisibility(View.GONE);
422            return;
423        }
424    }
425
426    public void onRobotAutomationEvent(String event) {
427        // Never show the latency chart during automated runs.
428        shouldShowLatencyChart = false;
429        // Disable full-screen mode to prevent modal user dialog.
430        enableFullScreen = false;
431        if (event.equals(RobotAutomationListener.RESTART_EVENT)) {
432            onClick(stopButton);
433        } else if (event.equals(RobotAutomationListener.START_EVENT)) {
434            onClick(startButton);
435        }
436    }
437
438    private WaltDevice.TriggerHandler brightnessTriggerHandler = new WaltDevice.TriggerHandler() {
439        @Override
440        public void onReceive(WaltDevice.TriggerMessage tmsg) {
441            logger.log("ERROR: Brightness curve trigger got a trigger message, " +
442                    "this should never happen."
443            );
444        }
445
446        @Override
447        public void onReceiveRaw(String s) {
448            brightnessCurveData.append(s);
449            if (s.trim().equals("end")) {
450                // Remove the delayed callback and run it now
451                handler.removeCallbacks(finishBrightnessCurve);
452                handler.post(finishBrightnessCurve);
453            }
454        }
455    };
456
457    void startBrightnessCurve() {
458        try {
459            brightnessCurveData = new StringBuilder();
460            waltDevice.syncClock();
461            waltDevice.startListener();
462        } catch (IOException e) {
463            logger.log("Error starting test: " + e.getMessage());
464            isTestRunning = false;
465            startButton.setEnabled(true);
466            return;
467        }
468        setFullScreen(enableFullScreen);
469        blackBox.setText("");
470        blackBox.setBackgroundColor(Color.BLACK);
471        handler.postDelayed(startBrightness, enableFullScreen ? 1000 : CURVE_BLINK_TIME);
472    }
473
474    Runnable startBrightness = new Runnable() {
475        @Override
476        public void run() {
477            waltDevice.setTriggerHandler(brightnessTriggerHandler);
478            long tStart = waltDevice.clock.micros();
479
480            try {
481                waltDevice.command(WaltDevice.CMD_BRIGHTNESS_CURVE);
482            } catch (IOException e) {
483                logger.log("Error sending command CMD_BRIGHTNESS_CURVE: " + e.getMessage());
484                isTestRunning = false;
485                startButton.setEnabled(true);
486                return;
487            }
488
489            blackBox.setBackgroundColor(Color.WHITE);
490
491            logger.log("=== Screen brightness curve: ===\nt_start: " + tStart);
492
493            handler.postDelayed(finishBrightnessCurve, CURVE_TIMEOUT);
494
495            // Schedule the screen to flip back to black in CURVE_BLINK_TIME ms
496            handler.postDelayed(new Runnable() {
497                @Override
498                public void run() {
499                    long tBack = waltDevice.clock.micros();
500                    blackBox.setBackgroundColor(Color.BLACK);
501                    logger.log("t_back: " + tBack);
502
503                }
504            }, CURVE_BLINK_TIME);
505        }
506    };
507
508    Runnable finishBrightnessCurve = new Runnable() {
509        @Override
510        public void run() {
511            waltDevice.stopListener();
512            waltDevice.clearTriggerHandler();
513
514            // TODO: Add option to save this data into a separate file rather than the main log.
515            logger.log(brightnessCurveData.toString());
516            logger.log("=== End of screen brightness data ===");
517
518            blackBox.setText(logger.getLogText());
519            blackBox.setMovementMethod(new ScrollingMovementMethod());
520            blackBox.setBackgroundColor(color_gray);
521            isTestRunning = false;
522            startButton.setEnabled(true);
523            setFullScreen(false);
524            drawBrightnessChart();
525            LogUploader.uploadIfAutoEnabled(getContext());
526        }
527    };
528
529    private void drawBrightnessChart() {
530        final String brightnessCurveString = brightnessCurveData.toString();
531        List<Entry> entries = new ArrayList<>();
532
533        // "u" marks the start of the brightness curve data
534        int startIndex = brightnessCurveString.indexOf("u") + 1;
535        int endIndex = brightnessCurveString.indexOf("end");
536        if (endIndex == -1) endIndex = brightnessCurveString.length();
537
538        String[] brightnessStrings =
539                brightnessCurveString.substring(startIndex, endIndex).trim().split("\n");
540        for (String str : brightnessStrings) {
541            String[] arr = str.split(" ");
542            final float timestampMs = Integer.parseInt(arr[0]) / 1000f;
543            final float brightness = Integer.parseInt(arr[1]);
544            entries.add(new Entry(timestampMs, brightness));
545        }
546        LineDataSet dataSet = new LineDataSet(entries, "Brightness");
547        dataSet.setColor(Color.BLACK);
548        dataSet.setValueTextColor(Color.BLACK);
549        dataSet.setCircleColor(Color.BLACK);
550        dataSet.setCircleRadius(1.5f);
551        dataSet.setCircleColorHole(Color.DKGRAY);
552        LineData lineData = new LineData(dataSet);
553        brightnessChart.setData(lineData);
554        final Description desc = new Description();
555        desc.setText("Screen Brightness [digital level 0-1023] vs. Time [ms]");
556        desc.setTextSize(12f);
557        brightnessChart.setDescription(desc);
558        brightnessChart.getLegend().setEnabled(false);
559        brightnessChart.invalidate();
560        brightnessChartLayout.setVisibility(View.VISIBLE);
561    }
562
563    private void increaseScreenBrightness() {
564        final WindowManager.LayoutParams layoutParams = getActivity().getWindow().getAttributes();
565        layoutParams.screenBrightness = 1f;
566        getActivity().getWindow().setAttributes(layoutParams);
567    }
568
569    private void setFullScreen(boolean enable) {
570        final AppCompatActivity activity = (AppCompatActivity) getActivity();
571        final ActionBar actionBar = activity != null ? activity.getSupportActionBar() : null;
572        int newVisibility = 0;
573        if (enable) {
574            if (actionBar != null) actionBar.hide();
575            buttonBarView.setVisibility(View.GONE);
576            newVisibility |= View.SYSTEM_UI_FLAG_FULLSCREEN
577                    | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
578                    | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY;
579        } else {
580            if (actionBar != null) actionBar.show();
581            buttonBarView.setVisibility(View.VISIBLE);
582        }
583        if (activity != null) activity.getWindow().getDecorView().setSystemUiVisibility(newVisibility);
584    }
585}
586