1/* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package org.chromium.latency.walt; 18 19import android.content.BroadcastReceiver; 20import android.content.Context; 21import android.content.Intent; 22import android.graphics.Color; 23import android.os.Bundle; 24import android.os.Handler; 25import android.support.v4.app.Fragment; 26import android.support.v7.app.ActionBar; 27import android.support.v7.app.AlertDialog; 28import android.support.v7.app.AppCompatActivity; 29import android.text.method.ScrollingMovementMethod; 30import android.view.Choreographer; 31import android.view.LayoutInflater; 32import android.view.View; 33import android.view.ViewGroup; 34import android.view.WindowManager; 35import android.widget.ArrayAdapter; 36import android.widget.Spinner; 37import android.widget.TextView; 38 39import com.github.mikephil.charting.charts.LineChart; 40import com.github.mikephil.charting.components.Description; 41import com.github.mikephil.charting.data.Entry; 42import com.github.mikephil.charting.data.LineData; 43import com.github.mikephil.charting.data.LineDataSet; 44 45import java.io.IOException; 46import java.util.ArrayList; 47import java.util.List; 48import java.util.Locale; 49 50import static org.chromium.latency.walt.Utils.getBooleanPreference; 51import static org.chromium.latency.walt.Utils.getIntPreference; 52 53/** 54 * Measurement of screen response time when switching between black and white. 55 */ 56public class ScreenResponseFragment extends Fragment 57 implements View.OnClickListener, RobotAutomationListener { 58 59 private static final int CURVE_TIMEOUT = 1000; // milliseconds 60 private static final int CURVE_BLINK_TIME = 250; // milliseconds 61 private static final int W2B_INDEX = 0; 62 private static final int B2W_INDEX = 1; 63 private SimpleLogger logger; 64 private TraceLogger traceLogger = null; 65 private WaltDevice waltDevice; 66 private Handler handler = new Handler(); 67 private TextView blackBox; 68 private View startButton; 69 private View stopButton; 70 private Spinner spinner; 71 private LineChart brightnessChart; 72 private HistogramChart latencyChart; 73 private View brightnessChartLayout; 74 private View buttonBarView; 75 private FastPathSurfaceView fastSurfaceView; 76 private int timesToBlink; 77 private boolean shouldShowLatencyChart = false; 78 private boolean isTestRunning = false; 79 private boolean enableFullScreen = false; 80 private boolean isFastPathGraphics = false; 81 int initiatedBlinks = 0; 82 int detectedBlinks = 0; 83 boolean isBoxWhite = false; 84 long lastFrameStartTime; 85 long lastFrameCallbackTime; 86 long lastSetBackgroundTime; 87 ArrayList<Double> deltas_w2b = new ArrayList<>(); 88 ArrayList<Double> deltas_b2w = new ArrayList<>(); 89 ArrayList<Double> deltas = new ArrayList<>(); 90 private static final int color_gray = Color.argb(0xFF, 0xBB, 0xBB, 0xBB); 91 private StringBuilder brightnessCurveData; 92 93 private BroadcastReceiver logReceiver = new BroadcastReceiver() { 94 @Override 95 public void onReceive(Context context, Intent intent) { 96 if (!isTestRunning) { 97 String msg = intent.getStringExtra("message"); 98 blackBox.append(msg + "\n"); 99 } 100 } 101 }; 102 103 public ScreenResponseFragment() { 104 // Required empty public constructor 105 } 106 107 @Override 108 public View onCreateView(LayoutInflater inflater, ViewGroup container, 109 Bundle savedInstanceState) { 110 timesToBlink = getIntPreference(getContext(), R.string.preference_screen_blinks, 20); 111 shouldShowLatencyChart = getBooleanPreference(getContext(), R.string.preference_show_blink_histogram, true); 112 enableFullScreen = getBooleanPreference(getContext(), R.string.preference_screen_fullscreen, true); 113 if (getBooleanPreference(getContext(), R.string.preference_systrace, true)) { 114 traceLogger = TraceLogger.getInstance(); 115 } 116 waltDevice = WaltDevice.getInstance(getContext()); 117 logger = SimpleLogger.getInstance(getContext()); 118 119 // Inflate the layout for this fragment 120 final View view = inflater.inflate(R.layout.fragment_screen_response, container, false); 121 stopButton = view.findViewById(R.id.button_stop_screen_response); 122 startButton = view.findViewById(R.id.button_start_screen_response); 123 blackBox = (TextView) view.findViewById(R.id.txt_black_box_screen); 124 fastSurfaceView = (FastPathSurfaceView) view.findViewById(R.id.fast_path_surface); 125 spinner = (Spinner) view.findViewById(R.id.spinner_screen_response); 126 buttonBarView = view.findViewById(R.id.button_bar); 127 ArrayAdapter<CharSequence> modeAdapter = ArrayAdapter.createFromResource(getContext(), 128 R.array.screen_response_mode_array, android.R.layout.simple_spinner_item); 129 modeAdapter.setDropDownViewResource(R.layout.support_simple_spinner_dropdown_item); 130 spinner.setAdapter(modeAdapter); 131 stopButton.setEnabled(false); 132 blackBox.setMovementMethod(new ScrollingMovementMethod()); 133 brightnessChartLayout = view.findViewById(R.id.brightness_chart_layout); 134 view.findViewById(R.id.button_close_chart).setOnClickListener(this); 135 brightnessChart = (LineChart) view.findViewById(R.id.chart); 136 latencyChart = (HistogramChart) view.findViewById(R.id.latency_chart); 137 138 if (getBooleanPreference(getContext(), R.string.preference_auto_increase_brightness, true)) { 139 increaseScreenBrightness(); 140 } 141 return view; 142 } 143 144 @Override 145 public void onResume() { 146 super.onResume(); 147 logger.registerReceiver(logReceiver); 148 // Register this fragment class as the listener for some button clicks 149 startButton.setOnClickListener(this); 150 stopButton.setOnClickListener(this); 151 } 152 153 @Override 154 public void onPause() { 155 logger.unregisterReceiver(logReceiver); 156 super.onPause(); 157 } 158 159 void startBlinkLatency() { 160 setFullScreen(enableFullScreen); 161 deltas.clear(); 162 deltas_b2w.clear(); 163 deltas_w2b.clear(); 164 if (shouldShowLatencyChart) { 165 latencyChart.clearData(); 166 latencyChart.setVisibility(View.VISIBLE); 167 latencyChart.setLabel(W2B_INDEX, "White-to-black"); 168 latencyChart.setLabel(B2W_INDEX, "Black-to-white"); 169 } 170 initiatedBlinks = 0; 171 detectedBlinks = 0; 172 if (isFastPathGraphics) { 173 blackBox.setVisibility(View.GONE); 174 fastSurfaceView.setVisibility(View.VISIBLE); 175 fastSurfaceView.setRectColor(Color.WHITE); 176 } else { 177 blackBox.setText(""); 178 blackBox.setBackgroundColor(Color.WHITE); 179 } 180 isBoxWhite = true; 181 182 handler.postDelayed(startBlinking, enableFullScreen ? 800 : 300); 183 } 184 185 Runnable startBlinking = new Runnable() { 186 @Override 187 public void run() { 188 try { 189 // Check for PWM 190 WaltDevice.TriggerMessage tmsg = waltDevice.readTriggerMessage(WaltDevice.CMD_SEND_LAST_SCREEN); 191 logger.log("Blink count was: " + tmsg.count); 192 193 waltDevice.softReset(); 194 waltDevice.syncClock(); // Note, sync also sends CMD_RESET (but not simpleSync). 195 waltDevice.command(WaltDevice.CMD_AUTO_SCREEN_ON); 196 waltDevice.startListener(); 197 } catch (IOException e) { 198 logger.log("Error: " + e.getMessage()); 199 } 200 201 // Register a callback for triggers 202 waltDevice.setTriggerHandler(triggerHandler); 203 204 // post doBlink runnable 205 handler.postDelayed(doBlinkRunnable, 100); 206 } 207 }; 208 209 Runnable doBlinkRunnable = new Runnable() { 210 @Override 211 public void run() { 212 if (!isTestRunning) return; 213 logger.log("======\ndoBlink.run(), initiatedBlinks = " + initiatedBlinks + " detectedBlinks = " + detectedBlinks); 214 // Check if we saw some transitions without blinking, this would usually mean 215 // the screen has PWM enabled, warn and ask the user to turn it off. 216 if (initiatedBlinks == 0 && detectedBlinks > 1) { 217 logger.log("Unexpected blinks detected, probably PWM, turn it off"); 218 isTestRunning = false; 219 stopButton.setEnabled(false); 220 startButton.setEnabled(true); 221 showPwmDialog(); 222 return; 223 } 224 225 if (initiatedBlinks >= timesToBlink) { 226 isTestRunning = false; 227 finishAndShowStats(); 228 return; 229 } 230 231 // * 2 flip the screen, save time as last flip time (last flip direction?) 232 233 isBoxWhite = !isBoxWhite; 234 int nextColor = isBoxWhite ? Color.WHITE : Color.BLACK; 235 initiatedBlinks++; 236 if (traceLogger != null) { 237 traceLogger.log(RemoteClockInfo.microTime(), RemoteClockInfo.microTime() + 1000, 238 "Request-to-" + (isBoxWhite ? "white" : "black"), 239 "Application has called setBackgroundColor at start of bar"); 240 } 241 if (isFastPathGraphics) { 242 fastSurfaceView.setRectColor(nextColor); 243 } else { 244 blackBox.setBackgroundColor(nextColor); 245 } 246 lastSetBackgroundTime = waltDevice.clock.micros(); 247 248 // Set up a callback to run on next frame render to collect the timestamp 249 Choreographer.getInstance().postFrameCallback(new Choreographer.FrameCallback() { 250 @Override 251 public void doFrame(long frameTimeNanos) { 252 // frameTimeNanos is the time in nanoseconds when the frame started being 253 // rendered, in the nanoTime() timebase. 254 lastFrameStartTime = frameTimeNanos / 1000 - waltDevice.clock.baseTime; 255 lastFrameCallbackTime = System.nanoTime() / 1000 - waltDevice.clock.baseTime; 256 } 257 }); 258 259 // Repost doBlink to some far away time to blink again even if nothing arrives from 260 // Teensy. This callback will almost always get cancelled by onIncomingTimestamp() 261 handler.postDelayed(doBlinkRunnable, 550 + (long) (Math.random()*100)); 262 } 263 }; 264 265 private void showPwmDialog() { 266 AlertDialog.Builder builder = new AlertDialog.Builder(getActivity()); 267 builder.setMessage("Detected extra blinks, please set your brightness to max") 268 .setTitle("Unexpected Blinks") 269 .setPositiveButton("OK", null); 270 AlertDialog dialog = builder.create(); 271 dialog.show(); 272 } 273 274 private WaltDevice.TriggerHandler triggerHandler = new WaltDevice.TriggerHandler() { 275 @Override 276 public void onReceive(WaltDevice.TriggerMessage tmsg) { 277 // Remove the far away doBlink callback 278 handler.removeCallbacks(doBlinkRunnable); 279 280 detectedBlinks++; 281 logger.log("blink counts " + initiatedBlinks + " " + detectedBlinks); 282 if (initiatedBlinks == 0) { 283 if (detectedBlinks < 5) { 284 logger.log("got incoming but initiatedBlinks = 0"); 285 return; 286 } else { 287 logger.log("Looks like PWM is used for this screen, turn auto brightness off and set it to max brightness"); 288 showPwmDialog(); 289 return; 290 } 291 } 292 293 if (traceLogger != null) { 294 final long startTimeMicros = lastFrameStartTime + waltDevice.clock.baseTime; 295 final long finishTimeMicros = tmsg.t + waltDevice.clock.baseTime; 296 traceLogger.log(startTimeMicros, finishTimeMicros, 297 isBoxWhite ? "Black-to-white" : "White-to-black", 298 "Bar starts at beginning of frame and ends when photosensor detects blink"); 299 } 300 301 double dt = (tmsg.t - lastFrameStartTime) / 1000.; 302 deltas.add(dt); 303 if (isBoxWhite) { // Current color is the color we transitioned to 304 deltas_b2w.add(dt); 305 } else { 306 deltas_w2b.add(dt); 307 } 308 if (shouldShowLatencyChart) latencyChart.addEntry(isBoxWhite ? B2W_INDEX : W2B_INDEX, dt); 309 310 // Other times can be important, logging them to allow more detailed analysis 311 logger.log(String.format(Locale.US, 312 "Times [ms]: setBG:%.3f callback:%.3f physical:%.3f black2white:%d", 313 (lastSetBackgroundTime - lastFrameStartTime) / 1000.0, 314 (lastFrameCallbackTime - lastFrameStartTime) / 1000.0, 315 dt, 316 isBoxWhite ? 1 : 0 317 )); 318 if (traceLogger != null) { 319 traceLogger.log(lastFrameCallbackTime + waltDevice.clock.baseTime, 320 lastFrameCallbackTime + waltDevice.clock.baseTime + 1000, 321 isBoxWhite ? "FrameCallback Black-to-white" : "FrameCallback White-to-black", 322 "FrameCallback was called at start of bar"); 323 } 324 // Schedule another blink soon-ish 325 handler.postDelayed(doBlinkRunnable, 40 + (long) (Math.random()*20)); 326 } 327 }; 328 329 330 void finishAndShowStats() { 331 setFullScreen(false); 332 // Stop the USB listener 333 waltDevice.stopListener(); 334 335 // Unregister trigger handler 336 waltDevice.clearTriggerHandler(); 337 338 waltDevice.sendAndFlush(WaltDevice.CMD_AUTO_SCREEN_OFF); 339 340 waltDevice.checkDrift(); 341 342 // Show deltas and the median 343 /* // Debug printouts 344 logger.log("deltas = array(" + deltas.toString() + ")"); 345 logger.log("deltas_w2b = array(" + deltas_w2b.toString() + ")"); 346 logger.log("deltas_b2w = array(" + deltas_b2w.toString() + ")"); 347 */ 348 349 double median_b2w = Utils.median(deltas_b2w); 350 double median_w2b = Utils.median(deltas_w2b); 351 logger.log(String.format(Locale.US, 352 "\n-------------------------------\n" + 353 "Median screen response latencies (N=%d):\n" + 354 "Black to white: %.1f ms (N=%d)\n" + 355 "White to black: %.1f ms (N=%d)\n" + 356 "Average: %.1f ms\n" + 357 "-------------------------------\n", 358 deltas.size(), 359 median_b2w, deltas_b2w.size(), 360 median_w2b, deltas_w2b.size(), 361 (median_b2w + median_w2b) / 2 362 )); 363 364 if (traceLogger != null) traceLogger.flush(getContext()); 365 fastSurfaceView.setVisibility(View.GONE); 366 blackBox.setVisibility(View.VISIBLE); 367 blackBox.setText(logger.getLogText()); 368 blackBox.setMovementMethod(new ScrollingMovementMethod()); 369 blackBox.setBackgroundColor(color_gray); 370 stopButton.setEnabled(false); 371 startButton.setEnabled(true); 372 if (shouldShowLatencyChart) { 373 latencyChart.setLabel(W2B_INDEX, String.format(Locale.US, "White-to-black m=%.1f ms", median_w2b)); 374 latencyChart.setLabel(B2W_INDEX, String.format(Locale.US, "Black-to-white m=%.1f ms", median_b2w)); 375 } 376 LogUploader.uploadIfAutoEnabled(getContext()); 377 } 378 379 @Override 380 public void onClick(View v) { 381 if (v.getId() == R.id.button_stop_screen_response) { 382 isTestRunning = false; 383 handler.removeCallbacks(doBlinkRunnable); 384 handler.removeCallbacks(startBlinking); 385 finishAndShowStats(); 386 return; 387 } 388 389 if (v.getId() == R.id.button_start_screen_response) { 390 brightnessChartLayout.setVisibility(View.GONE); 391 latencyChart.setVisibility(View.GONE); 392 if (!waltDevice.isConnected()) { 393 logger.log("Error starting test: WALT is not connected"); 394 return; 395 } 396 397 isTestRunning = true; 398 startButton.setEnabled(false); 399 blackBox.setBackgroundColor(Color.BLACK); 400 blackBox.setText(""); 401 isFastPathGraphics = false; 402 final int spinnerPosition = spinner.getSelectedItemPosition(); 403 if (spinnerPosition == 0) { 404 logger.log("Starting screen response measurement"); 405 stopButton.setEnabled(true); 406 startBlinkLatency(); 407 } else if (spinnerPosition == 1) { 408 logger.log("Starting screen brightness curve measurement"); 409 startBrightnessCurve(); 410 } else if (spinnerPosition == 2) { 411 logger.log("Starting fast-path screen response measurement"); 412 isFastPathGraphics = true; 413 startBlinkLatency(); 414 } else { 415 logger.log("ERROR: Spinner position is out of range"); 416 } 417 return; 418 } 419 420 if (v.getId() == R.id.button_close_chart) { 421 brightnessChartLayout.setVisibility(View.GONE); 422 return; 423 } 424 } 425 426 public void onRobotAutomationEvent(String event) { 427 // Never show the latency chart during automated runs. 428 shouldShowLatencyChart = false; 429 // Disable full-screen mode to prevent modal user dialog. 430 enableFullScreen = false; 431 if (event.equals(RobotAutomationListener.RESTART_EVENT)) { 432 onClick(stopButton); 433 } else if (event.equals(RobotAutomationListener.START_EVENT)) { 434 onClick(startButton); 435 } 436 } 437 438 private WaltDevice.TriggerHandler brightnessTriggerHandler = new WaltDevice.TriggerHandler() { 439 @Override 440 public void onReceive(WaltDevice.TriggerMessage tmsg) { 441 logger.log("ERROR: Brightness curve trigger got a trigger message, " + 442 "this should never happen." 443 ); 444 } 445 446 @Override 447 public void onReceiveRaw(String s) { 448 brightnessCurveData.append(s); 449 if (s.trim().equals("end")) { 450 // Remove the delayed callback and run it now 451 handler.removeCallbacks(finishBrightnessCurve); 452 handler.post(finishBrightnessCurve); 453 } 454 } 455 }; 456 457 void startBrightnessCurve() { 458 try { 459 brightnessCurveData = new StringBuilder(); 460 waltDevice.syncClock(); 461 waltDevice.startListener(); 462 } catch (IOException e) { 463 logger.log("Error starting test: " + e.getMessage()); 464 isTestRunning = false; 465 startButton.setEnabled(true); 466 return; 467 } 468 setFullScreen(enableFullScreen); 469 blackBox.setText(""); 470 blackBox.setBackgroundColor(Color.BLACK); 471 handler.postDelayed(startBrightness, enableFullScreen ? 1000 : CURVE_BLINK_TIME); 472 } 473 474 Runnable startBrightness = new Runnable() { 475 @Override 476 public void run() { 477 waltDevice.setTriggerHandler(brightnessTriggerHandler); 478 long tStart = waltDevice.clock.micros(); 479 480 try { 481 waltDevice.command(WaltDevice.CMD_BRIGHTNESS_CURVE); 482 } catch (IOException e) { 483 logger.log("Error sending command CMD_BRIGHTNESS_CURVE: " + e.getMessage()); 484 isTestRunning = false; 485 startButton.setEnabled(true); 486 return; 487 } 488 489 blackBox.setBackgroundColor(Color.WHITE); 490 491 logger.log("=== Screen brightness curve: ===\nt_start: " + tStart); 492 493 handler.postDelayed(finishBrightnessCurve, CURVE_TIMEOUT); 494 495 // Schedule the screen to flip back to black in CURVE_BLINK_TIME ms 496 handler.postDelayed(new Runnable() { 497 @Override 498 public void run() { 499 long tBack = waltDevice.clock.micros(); 500 blackBox.setBackgroundColor(Color.BLACK); 501 logger.log("t_back: " + tBack); 502 503 } 504 }, CURVE_BLINK_TIME); 505 } 506 }; 507 508 Runnable finishBrightnessCurve = new Runnable() { 509 @Override 510 public void run() { 511 waltDevice.stopListener(); 512 waltDevice.clearTriggerHandler(); 513 514 // TODO: Add option to save this data into a separate file rather than the main log. 515 logger.log(brightnessCurveData.toString()); 516 logger.log("=== End of screen brightness data ==="); 517 518 blackBox.setText(logger.getLogText()); 519 blackBox.setMovementMethod(new ScrollingMovementMethod()); 520 blackBox.setBackgroundColor(color_gray); 521 isTestRunning = false; 522 startButton.setEnabled(true); 523 setFullScreen(false); 524 drawBrightnessChart(); 525 LogUploader.uploadIfAutoEnabled(getContext()); 526 } 527 }; 528 529 private void drawBrightnessChart() { 530 final String brightnessCurveString = brightnessCurveData.toString(); 531 List<Entry> entries = new ArrayList<>(); 532 533 // "u" marks the start of the brightness curve data 534 int startIndex = brightnessCurveString.indexOf("u") + 1; 535 int endIndex = brightnessCurveString.indexOf("end"); 536 if (endIndex == -1) endIndex = brightnessCurveString.length(); 537 538 String[] brightnessStrings = 539 brightnessCurveString.substring(startIndex, endIndex).trim().split("\n"); 540 for (String str : brightnessStrings) { 541 String[] arr = str.split(" "); 542 final float timestampMs = Integer.parseInt(arr[0]) / 1000f; 543 final float brightness = Integer.parseInt(arr[1]); 544 entries.add(new Entry(timestampMs, brightness)); 545 } 546 LineDataSet dataSet = new LineDataSet(entries, "Brightness"); 547 dataSet.setColor(Color.BLACK); 548 dataSet.setValueTextColor(Color.BLACK); 549 dataSet.setCircleColor(Color.BLACK); 550 dataSet.setCircleRadius(1.5f); 551 dataSet.setCircleColorHole(Color.DKGRAY); 552 LineData lineData = new LineData(dataSet); 553 brightnessChart.setData(lineData); 554 final Description desc = new Description(); 555 desc.setText("Screen Brightness [digital level 0-1023] vs. Time [ms]"); 556 desc.setTextSize(12f); 557 brightnessChart.setDescription(desc); 558 brightnessChart.getLegend().setEnabled(false); 559 brightnessChart.invalidate(); 560 brightnessChartLayout.setVisibility(View.VISIBLE); 561 } 562 563 private void increaseScreenBrightness() { 564 final WindowManager.LayoutParams layoutParams = getActivity().getWindow().getAttributes(); 565 layoutParams.screenBrightness = 1f; 566 getActivity().getWindow().setAttributes(layoutParams); 567 } 568 569 private void setFullScreen(boolean enable) { 570 final AppCompatActivity activity = (AppCompatActivity) getActivity(); 571 final ActionBar actionBar = activity != null ? activity.getSupportActionBar() : null; 572 int newVisibility = 0; 573 if (enable) { 574 if (actionBar != null) actionBar.hide(); 575 buttonBarView.setVisibility(View.GONE); 576 newVisibility |= View.SYSTEM_UI_FLAG_FULLSCREEN 577 | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION 578 | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY; 579 } else { 580 if (actionBar != null) actionBar.show(); 581 buttonBarView.setVisibility(View.VISIBLE); 582 } 583 if (activity != null) activity.getWindow().getDecorView().setSystemUiVisibility(newVisibility); 584 } 585} 586