1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "win8/test/ui_automation_client.h"
6
7#include <atlbase.h>
8#include <atlcom.h>
9#include <oleauto.h>
10#include <uiautomation.h>
11
12#include <algorithm>
13
14#include "base/bind.h"
15#include "base/callback.h"
16#include "base/memory/ref_counted.h"
17#include "base/strings/string_number_conversions.h"
18#include "base/strings/string_util.h"
19#include "base/thread_task_runner_handle.h"
20#include "base/win/scoped_comptr.h"
21#include "base/win/scoped_variant.h"
22
23namespace win8 {
24namespace internal {
25
26// The guts of the UI automation client which runs on a dedicated thread in the
27// multi-threaded COM apartment. An instance may be constructed on any thread,
28// but Initialize() must be invoked on a thread in the MTA.
29class UIAutomationClient::Context {
30 public:
31  // Returns a new instance ready for initialization and use on another thread.
32  static base::WeakPtr<Context> Create();
33
34  // Deletes the instance.
35  void DeleteOnAutomationThread();
36
37  // Initializes the context, invoking |init_callback| via |client_runner| when
38  // done. On success, |result_callback| will eventually be called after the
39  // window has been processed. On failure, this instance self-destructs after
40  // posting |init_callback|.
41  void Initialize(
42      scoped_refptr<base::SingleThreadTaskRunner> client_runner,
43      string16 class_name,
44      string16 item_name,
45      UIAutomationClient::InitializedCallback init_callback,
46      UIAutomationClient::ResultCallback result_callback);
47
48  // Methods invoked by event handlers via weak pointers.
49  void HandleAutomationEvent(
50      base::win::ScopedComPtr<IUIAutomationElement> sender,
51      EVENTID eventId);
52
53 private:
54  class EventHandler;
55
56  // The only and only method that may be called from outside of the automation
57  // thread.
58  Context();
59  ~Context();
60
61  HRESULT InstallWindowObserver();
62  HRESULT RemoveWindowObserver();
63
64  void HandleWindowOpen(
65      const base::win::ScopedComPtr<IUIAutomationElement>& window);
66  void ProcessWindow(
67      const base::win::ScopedComPtr<IUIAutomationElement>& window);
68  HRESULT InvokeDesiredItem(
69      const base::win::ScopedComPtr<IUIAutomationElement>& element);
70  HRESULT GetInvokableItems(
71      const base::win::ScopedComPtr<IUIAutomationElement>& element,
72      std::vector<string16>* choices);
73  void CloseWindow(const base::win::ScopedComPtr<IUIAutomationElement>& window);
74
75  base::ThreadChecker thread_checker_;
76
77  // The loop on which the client itself lives.
78  scoped_refptr<base::SingleThreadTaskRunner> client_runner_;
79
80  // The class name of the window for which the client waits.
81  string16 class_name_;
82
83  // The name of the item to invoke.
84  string16 item_name_;
85
86  // The consumer's result callback.
87  ResultCallback result_callback_;
88
89  // The automation client.
90  base::win::ScopedComPtr<IUIAutomation> automation_;
91
92  // A handler of Window open events.
93  base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler_;
94
95  // Weak pointers to the context are given to event handlers.
96  base::WeakPtrFactory<UIAutomationClient::Context> weak_ptr_factory_;
97
98  DISALLOW_COPY_AND_ASSIGN(Context);
99};
100
101class UIAutomationClient::Context::EventHandler
102    : public CComObjectRootEx<CComMultiThreadModel>,
103      public IUIAutomationEventHandler {
104 public:
105  BEGIN_COM_MAP(UIAutomationClient::Context::EventHandler)
106    COM_INTERFACE_ENTRY(IUIAutomationEventHandler)
107  END_COM_MAP()
108
109  EventHandler();
110  virtual ~EventHandler();
111
112  // Initializes the object with its parent UI automation client context's
113  // message loop and pointer. Events are dispatched back to the context on
114  // the given loop.
115  void Initialize(
116      const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
117      const base::WeakPtr<UIAutomationClient::Context>& context);
118
119  // IUIAutomationEventHandler methods.
120  STDMETHOD(HandleAutomationEvent)(IUIAutomationElement* sender,
121                                   EVENTID eventId);
122
123 private:
124  // The task runner for the UI automation client context.
125  scoped_refptr<base::SingleThreadTaskRunner> context_runner_;
126
127  // The parent UI automation client context.
128  base::WeakPtr<UIAutomationClient::Context> context_;
129
130  DISALLOW_COPY_AND_ASSIGN(EventHandler);
131};
132
133UIAutomationClient::Context::EventHandler::EventHandler() {}
134
135UIAutomationClient::Context::EventHandler::~EventHandler() {}
136
137void UIAutomationClient::Context::EventHandler::Initialize(
138    const scoped_refptr<base::SingleThreadTaskRunner>& context_runner,
139    const base::WeakPtr<UIAutomationClient::Context>& context) {
140  context_runner_ = context_runner;
141  context_ = context;
142}
143
144HRESULT UIAutomationClient::Context::EventHandler::HandleAutomationEvent(
145    IUIAutomationElement* sender,
146    EVENTID eventId) {
147  // Event handlers are invoked on an arbitrary thread in the MTA. Send the
148  // event back to the main UI automation thread for processing.
149  context_runner_->PostTask(
150      FROM_HERE,
151      base::Bind(&UIAutomationClient::Context::HandleAutomationEvent, context_,
152                 base::win::ScopedComPtr<IUIAutomationElement>(sender),
153                 eventId));
154
155  return S_OK;
156}
157
158base::WeakPtr<UIAutomationClient::Context>
159    UIAutomationClient::Context::Create() {
160  Context* context = new Context();
161  return context->weak_ptr_factory_.GetWeakPtr();
162}
163
164void UIAutomationClient::Context::DeleteOnAutomationThread() {
165  DCHECK(thread_checker_.CalledOnValidThread());
166  delete this;
167}
168
169UIAutomationClient::Context::Context() : weak_ptr_factory_(this) {}
170
171UIAutomationClient::Context::~Context() {
172  DCHECK(thread_checker_.CalledOnValidThread());
173
174  if (event_handler_.get()) {
175    event_handler_ = NULL;
176    HRESULT result = automation_->RemoveAllEventHandlers();
177    LOG_IF(ERROR, FAILED(result)) << std::hex << result;
178  }
179}
180
181void UIAutomationClient::Context::Initialize(
182    scoped_refptr<base::SingleThreadTaskRunner> client_runner,
183    string16 class_name,
184    string16 item_name,
185    UIAutomationClient::InitializedCallback init_callback,
186    UIAutomationClient::ResultCallback result_callback) {
187  // This and all other methods must be called on the automation thread.
188  DCHECK(!client_runner->BelongsToCurrentThread());
189  // Bind the checker to this thread.
190  thread_checker_.DetachFromThread();
191  DCHECK(thread_checker_.CalledOnValidThread());
192
193  client_runner_ = client_runner;
194  class_name_ = class_name;
195  item_name_ = item_name;
196  result_callback_ = result_callback;
197
198  HRESULT result = automation_.CreateInstance(CLSID_CUIAutomation, NULL,
199                                              CLSCTX_INPROC_SERVER);
200  if (FAILED(result) || !automation_.get())
201    LOG(ERROR) << std::hex << result;
202  else
203    result = InstallWindowObserver();
204
205  // Tell the client that initialization is complete.
206  client_runner_->PostTask(FROM_HERE, base::Bind(init_callback, result));
207
208  // Self-destruct if the overall operation failed.
209  if (FAILED(result))
210    delete this;
211}
212
213// Installs the window observer.
214HRESULT UIAutomationClient::Context::InstallWindowObserver() {
215  DCHECK(thread_checker_.CalledOnValidThread());
216  DCHECK(automation_.get());
217  DCHECK(!event_handler_.get());
218
219  HRESULT result = S_OK;
220  base::win::ScopedComPtr<IUIAutomationElement> root_element;
221  base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
222
223  // Observe the opening of all windows.
224  result = automation_->GetRootElement(root_element.Receive());
225  if (FAILED(result)) {
226    LOG(ERROR) << std::hex << result;
227    return result;
228  }
229
230  // Cache Window class, HWND, and window pattern for opened windows.
231  result = automation_->CreateCacheRequest(cache_request.Receive());
232  if (FAILED(result)) {
233    LOG(ERROR) << std::hex << result;
234    return result;
235  }
236  cache_request->AddProperty(UIA_ClassNamePropertyId);
237  cache_request->AddProperty(UIA_NativeWindowHandlePropertyId);
238
239  // Create the observer.
240  CComObject<EventHandler>* event_handler_obj = NULL;
241  result = CComObject<EventHandler>::CreateInstance(&event_handler_obj);
242  if (FAILED(result)) {
243    LOG(ERROR) << std::hex << result;
244    return result;
245  }
246  event_handler_obj->Initialize(base::ThreadTaskRunnerHandle::Get(),
247                                weak_ptr_factory_.GetWeakPtr());
248  base::win::ScopedComPtr<IUIAutomationEventHandler> event_handler(
249      event_handler_obj);
250
251  result = automation_->AddAutomationEventHandler(
252      UIA_Window_WindowOpenedEventId,
253      root_element,
254      TreeScope_Descendants,
255      cache_request,
256      event_handler);
257
258  if (FAILED(result)) {
259    LOG(ERROR) << std::hex << result;
260    return result;
261  }
262
263  event_handler_ = event_handler;
264  return S_OK;
265}
266
267// Removes this instance's window observer.
268HRESULT UIAutomationClient::Context::RemoveWindowObserver() {
269  DCHECK(thread_checker_.CalledOnValidThread());
270  DCHECK(automation_.get());
271  DCHECK(event_handler_.get());
272
273  HRESULT result = S_OK;
274  base::win::ScopedComPtr<IUIAutomationElement> root_element;
275
276  // The opening of all windows are observed.
277  result = automation_->GetRootElement(root_element.Receive());
278  if (FAILED(result)) {
279    LOG(ERROR) << std::hex << result;
280    return result;
281  }
282
283  result = automation_->RemoveAutomationEventHandler(
284      UIA_Window_WindowOpenedEventId,
285      root_element,
286      event_handler_);
287  if (FAILED(result)) {
288    LOG(ERROR) << std::hex << result;
289    return result;
290  }
291
292  event_handler_ = NULL;
293  return S_OK;
294}
295
296// Handles an automation event. If the event results in the processing for which
297// this context was created, the context self-destructs after posting the
298// results to the client.
299void UIAutomationClient::Context::HandleAutomationEvent(
300    base::win::ScopedComPtr<IUIAutomationElement> sender,
301    EVENTID eventId) {
302  DCHECK(thread_checker_.CalledOnValidThread());
303  if (eventId == UIA_Window_WindowOpenedEventId)
304    HandleWindowOpen(sender);
305}
306
307// Handles a WindowOpen event. If |window| is the one for which this instance is
308// waiting, it is processed and this instance self-destructs after posting the
309// results to the client.
310void UIAutomationClient::Context::HandleWindowOpen(
311    const base::win::ScopedComPtr<IUIAutomationElement>& window) {
312  DCHECK(thread_checker_.CalledOnValidThread());
313  HRESULT hr = S_OK;
314  base::win::ScopedVariant var;
315
316  hr = window->GetCachedPropertyValueEx(UIA_ClassNamePropertyId, TRUE,
317                                        var.Receive());
318  if (FAILED(hr)) {
319    LOG(ERROR) << std::hex << hr;
320    return;
321  }
322
323  if (V_VT(&var) != VT_BSTR) {
324    LOG(ERROR) << __FUNCTION__ " class name is not a BSTR: " << V_VT(&var);
325    return;
326  }
327
328  string16 class_name(V_BSTR(&var));
329
330  // Window class names are atoms, which are case-insensitive.
331  if (class_name.size() == class_name_.size() &&
332      std::equal(class_name.begin(), class_name.end(), class_name_.begin(),
333                 base::CaseInsensitiveCompare<wchar_t>())) {
334    RemoveWindowObserver();
335    ProcessWindow(window);
336  }
337}
338
339// Processes |window| by invoking the desired child item. If the item cannot be
340// found or invoked, an attempt is made to get a list of all invokable children.
341// The results are posted back to the client on |client_runner_|, and this
342// instance self-destructs.
343void UIAutomationClient::Context::ProcessWindow(
344    const base::win::ScopedComPtr<IUIAutomationElement>& window) {
345  DCHECK(thread_checker_.CalledOnValidThread());
346
347  HRESULT result = S_OK;
348  std::vector<string16> choices;
349  result = InvokeDesiredItem(window);
350  if (FAILED(result)) {
351    GetInvokableItems(window, &choices);
352    CloseWindow(window);
353  }
354
355  client_runner_->PostTask(FROM_HERE,
356                           base::Bind(result_callback_, result, choices));
357
358  // Self-destruct since there's nothing more to be done here.
359  delete this;
360}
361
362// Invokes the desired child of |element|.
363HRESULT UIAutomationClient::Context::InvokeDesiredItem(
364    const base::win::ScopedComPtr<IUIAutomationElement>& element) {
365  DCHECK(thread_checker_.CalledOnValidThread());
366
367  HRESULT result = S_OK;
368  base::win::ScopedVariant var;
369  base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
370  base::win::ScopedComPtr<IUIAutomationCondition> item_name_condition;
371  base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
372  base::win::ScopedComPtr<IUIAutomationCondition> condition;
373  base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
374  base::win::ScopedComPtr<IUIAutomationElement> target;
375
376  // Search for an invokable element named item_name.
377  var.Set(true);
378  result = automation_->CreatePropertyCondition(
379      UIA_IsInvokePatternAvailablePropertyId,
380      var,
381      invokable_condition.Receive());
382  var.Reset();
383  if (FAILED(result)) {
384    LOG(ERROR) << std::hex << result;
385    return false;
386  }
387
388  var.Set(item_name_.c_str());
389  result = automation_->CreatePropertyCondition(UIA_NamePropertyId,
390                                                var,
391                                                item_name_condition.Receive());
392  var.Reset();
393  if (FAILED(result)) {
394    LOG(ERROR) << std::hex << result;
395    return result;
396  }
397
398  result = automation_->get_ControlViewCondition(
399      control_view_condition.Receive());
400  if (FAILED(result)) {
401    LOG(ERROR) << std::hex << result;
402    return result;
403  }
404
405  std::vector<IUIAutomationCondition*> conditions;
406  conditions.push_back(invokable_condition.get());
407  conditions.push_back(item_name_condition.get());
408  conditions.push_back(control_view_condition.get());
409  result = automation_->CreateAndConditionFromNativeArray(
410      &conditions[0], conditions.size(), condition.Receive());
411  if (FAILED(result)) {
412    LOG(ERROR) << std::hex << result;
413    return result;
414  }
415
416  // Cache invokable pattern for the item.
417  result = automation_->CreateCacheRequest(cache_request.Receive());
418  if (FAILED(result)) {
419    LOG(ERROR) << std::hex << result;
420    return result;
421  }
422  cache_request->AddPattern(UIA_InvokePatternId);
423
424  result = element->FindFirstBuildCache(
425      static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
426      condition,
427      cache_request,
428      target.Receive());
429  if (FAILED(result)) {
430    LOG(ERROR) << std::hex << result;
431    return result;
432  }
433
434  // If the item was found, invoke it.
435  if (!target.get()) {
436    LOG(WARNING) << "Failed to find desired item to invoke.";
437    return E_FAIL;
438  }
439
440  base::win::ScopedComPtr<IUIAutomationInvokePattern> invoker;
441  result = target->GetCachedPatternAs(UIA_InvokePatternId, invoker.iid(),
442                                      invoker.ReceiveVoid());
443  if (FAILED(result)) {
444    LOG(ERROR) << std::hex << result;
445    return result;
446  }
447
448  result = invoker->Invoke();
449  if (FAILED(result)) {
450    LOG(ERROR) << std::hex << result;
451    return result;
452  }
453
454  return S_OK;
455}
456
457// Populates |choices| with the names of all invokable children of |element|.
458HRESULT UIAutomationClient::Context::GetInvokableItems(
459    const base::win::ScopedComPtr<IUIAutomationElement>& element,
460    std::vector<string16>* choices) {
461  DCHECK(choices);
462  DCHECK(thread_checker_.CalledOnValidThread());
463
464  HRESULT result = S_OK;
465  base::win::ScopedVariant var;
466  base::win::ScopedComPtr<IUIAutomationCondition> invokable_condition;
467  base::win::ScopedComPtr<IUIAutomationCondition> control_view_condition;
468  base::win::ScopedComPtr<IUIAutomationCondition> condition;
469  base::win::ScopedComPtr<IUIAutomationCacheRequest> cache_request;
470  base::win::ScopedComPtr<IUIAutomationElementArray> element_array;
471  base::win::ScopedComPtr<IUIAutomationElement> child_element;
472
473  // Search for all invokable elements.
474  var.Set(true);
475  result = automation_->CreatePropertyCondition(
476      UIA_IsInvokePatternAvailablePropertyId,
477      var,
478      invokable_condition.Receive());
479  var.Reset();
480  if (FAILED(result)) {
481    LOG(ERROR) << std::hex << result;
482    return result;
483  }
484
485  result = automation_->get_ControlViewCondition(
486      control_view_condition.Receive());
487  if (FAILED(result)) {
488    LOG(ERROR) << std::hex << result;
489    return result;
490  }
491
492  result = automation_->CreateAndCondition(
493      invokable_condition, control_view_condition, condition.Receive());
494  if (FAILED(result)) {
495    LOG(ERROR) << std::hex << result;
496    return result;
497  }
498
499  // Cache item names.
500  result = automation_->CreateCacheRequest(cache_request.Receive());
501  if (FAILED(result)) {
502    LOG(ERROR) << std::hex << result;
503    return result;
504  }
505  cache_request->AddProperty(UIA_NamePropertyId);
506
507  result = element->FindAllBuildCache(
508      static_cast<TreeScope>(TreeScope_Children | TreeScope_Descendants),
509      condition,
510      cache_request,
511      element_array.Receive());
512  if (FAILED(result)) {
513    LOG(ERROR) << std::hex << result;
514    return result;
515  }
516
517  if (!element_array.get()) {
518    LOG(ERROR) << "The window may have vanished.";
519    return S_OK;
520  }
521
522  int num_elements = 0;
523  result = element_array->get_Length(&num_elements);
524  if (FAILED(result)) {
525    LOG(ERROR) << std::hex << result;
526    return result;
527  }
528
529  choices->clear();
530  choices->reserve(num_elements);
531  for (int i = 0; i < num_elements; ++i) {
532    child_element.Release();
533    result = element_array->GetElement(i, child_element.Receive());
534    if (FAILED(result)) {
535      LOG(ERROR) << std::hex << result;
536      continue;
537    }
538    result = child_element->GetCachedPropertyValueEx(UIA_NamePropertyId, TRUE,
539                                                     var.Receive());
540    if (FAILED(result)) {
541      LOG(ERROR) << std::hex << result;
542      continue;
543    }
544    if (V_VT(&var) != VT_BSTR) {
545      LOG(ERROR) << __FUNCTION__ " name is not a BSTR: " << V_VT(&var);
546      continue;
547    }
548    choices->push_back(string16(V_BSTR(&var)));
549    var.Reset();
550  }
551
552  return result;
553}
554
555// Closes the element |window| by sending it an escape key.
556void UIAutomationClient::Context::CloseWindow(
557    const base::win::ScopedComPtr<IUIAutomationElement>& window) {
558  DCHECK(thread_checker_.CalledOnValidThread());
559
560  // It's tempting to get the Window pattern from |window| and invoke its Close
561  // method. Unfortunately, this doesn't work. Sending an escape key does the
562  // trick, though.
563  HRESULT result = S_OK;
564  base::win::ScopedVariant var;
565
566  result = window->GetCachedPropertyValueEx(
567      UIA_NativeWindowHandlePropertyId,
568      TRUE,
569      var.Receive());
570  if (FAILED(result)) {
571    LOG(ERROR) << std::hex << result;
572    return;
573  }
574
575  if (V_VT(&var) != VT_I4) {
576    LOG(ERROR) << __FUNCTION__ " window handle is not an int: " << V_VT(&var);
577    return;
578  }
579
580  HWND handle = reinterpret_cast<HWND>(V_I4(&var));
581
582  uint32 scan_code = MapVirtualKey(VK_ESCAPE, MAPVK_VK_TO_VSC);
583  PostMessage(handle, WM_KEYDOWN, VK_ESCAPE,
584              MAKELPARAM(1, scan_code));
585  PostMessage(handle, WM_KEYUP, VK_ESCAPE,
586              MAKELPARAM(1, scan_code | KF_REPEAT | KF_UP));
587}
588
589UIAutomationClient::UIAutomationClient()
590    : automation_thread_("UIAutomation") {}
591
592UIAutomationClient::~UIAutomationClient() {
593  DCHECK(thread_checker_.CalledOnValidThread());
594
595  // context_ is still valid when the caller destroys the instance before the
596  // callback(s) have fired. In this case, delete the context on the automation
597  // thread before joining with it.
598  automation_thread_.message_loop()->PostTask(
599      FROM_HERE,
600      base::Bind(&UIAutomationClient::Context::DeleteOnAutomationThread,
601                 context_));
602}
603
604void UIAutomationClient::Begin(const wchar_t* class_name,
605                               const string16& item_name,
606                               const InitializedCallback& init_callback,
607                               const ResultCallback& result_callback) {
608  DCHECK(thread_checker_.CalledOnValidThread());
609  DCHECK_EQ(context_.get(), static_cast<Context*>(NULL));
610
611  // Start the automation thread and initialize our automation client on it.
612  context_ = Context::Create();
613  automation_thread_.init_com_with_mta(true);
614  automation_thread_.Start();
615  automation_thread_.message_loop()->PostTask(
616      FROM_HERE,
617      base::Bind(&UIAutomationClient::Context::Initialize,
618                 context_,
619                 base::ThreadTaskRunnerHandle::Get(),
620                 string16(class_name),
621                 item_name,
622                 init_callback,
623                 result_callback));
624}
625
626}  // namespace internal
627}  // namespace win8
628