1/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_mips64.h"
18#include "mirror/array-inl.h"
19
20namespace art {
21namespace mips64 {
22
23// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
24#define __ down_cast<Mips64Assembler*>(GetAssembler())->  // NOLINT
25
26VectorRegister VectorRegisterFrom(Location location) {
27  DCHECK(location.IsFpuRegister());
28  return static_cast<VectorRegister>(location.AsFpuRegister<FpuRegister>());
29}
30
31void LocationsBuilderMIPS64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
32  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
33  switch (instruction->GetPackedType()) {
34    case DataType::Type::kBool:
35    case DataType::Type::kUint8:
36    case DataType::Type::kInt8:
37    case DataType::Type::kUint16:
38    case DataType::Type::kInt16:
39    case DataType::Type::kInt32:
40    case DataType::Type::kInt64:
41      locations->SetInAt(0, Location::RequiresRegister());
42      locations->SetOut(Location::RequiresFpuRegister());
43      break;
44    case DataType::Type::kFloat32:
45    case DataType::Type::kFloat64:
46      locations->SetInAt(0, Location::RequiresFpuRegister());
47      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
48      break;
49    default:
50      LOG(FATAL) << "Unsupported SIMD type";
51      UNREACHABLE();
52  }
53}
54
55void InstructionCodeGeneratorMIPS64::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
56  LocationSummary* locations = instruction->GetLocations();
57  VectorRegister dst = VectorRegisterFrom(locations->Out());
58  switch (instruction->GetPackedType()) {
59    case DataType::Type::kBool:
60    case DataType::Type::kUint8:
61    case DataType::Type::kInt8:
62      DCHECK_EQ(16u, instruction->GetVectorLength());
63      __ FillB(dst, locations->InAt(0).AsRegister<GpuRegister>());
64      break;
65    case DataType::Type::kUint16:
66    case DataType::Type::kInt16:
67      DCHECK_EQ(8u, instruction->GetVectorLength());
68      __ FillH(dst, locations->InAt(0).AsRegister<GpuRegister>());
69      break;
70    case DataType::Type::kInt32:
71      DCHECK_EQ(4u, instruction->GetVectorLength());
72      __ FillW(dst, locations->InAt(0).AsRegister<GpuRegister>());
73      break;
74    case DataType::Type::kInt64:
75      DCHECK_EQ(2u, instruction->GetVectorLength());
76      __ FillD(dst, locations->InAt(0).AsRegister<GpuRegister>());
77      break;
78    case DataType::Type::kFloat32:
79      DCHECK_EQ(4u, instruction->GetVectorLength());
80      __ ReplicateFPToVectorRegister(dst,
81                                     locations->InAt(0).AsFpuRegister<FpuRegister>(),
82                                     /* is_double */ false);
83      break;
84    case DataType::Type::kFloat64:
85      DCHECK_EQ(2u, instruction->GetVectorLength());
86      __ ReplicateFPToVectorRegister(dst,
87                                     locations->InAt(0).AsFpuRegister<FpuRegister>(),
88                                     /* is_double */ true);
89      break;
90    default:
91      LOG(FATAL) << "Unsupported SIMD type";
92      UNREACHABLE();
93  }
94}
95
96void LocationsBuilderMIPS64::VisitVecExtractScalar(HVecExtractScalar* instruction) {
97  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
98  switch (instruction->GetPackedType()) {
99    case DataType::Type::kBool:
100    case DataType::Type::kUint8:
101    case DataType::Type::kInt8:
102    case DataType::Type::kUint16:
103    case DataType::Type::kInt16:
104    case DataType::Type::kInt32:
105    case DataType::Type::kInt64:
106      locations->SetInAt(0, Location::RequiresFpuRegister());
107      locations->SetOut(Location::RequiresRegister());
108      break;
109    case DataType::Type::kFloat32:
110    case DataType::Type::kFloat64:
111      locations->SetInAt(0, Location::RequiresFpuRegister());
112      locations->SetOut(Location::SameAsFirstInput());
113      break;
114    default:
115      LOG(FATAL) << "Unsupported SIMD type";
116      UNREACHABLE();
117  }
118}
119
120void InstructionCodeGeneratorMIPS64::VisitVecExtractScalar(HVecExtractScalar* instruction) {
121  LocationSummary* locations = instruction->GetLocations();
122  VectorRegister src = VectorRegisterFrom(locations->InAt(0));
123  switch (instruction->GetPackedType()) {
124    case DataType::Type::kInt32:
125      DCHECK_EQ(4u, instruction->GetVectorLength());
126      __ Copy_sW(locations->Out().AsRegister<GpuRegister>(), src, 0);
127      break;
128    case DataType::Type::kInt64:
129      DCHECK_EQ(2u, instruction->GetVectorLength());
130      __ Copy_sD(locations->Out().AsRegister<GpuRegister>(), src, 0);
131      break;
132    case DataType::Type::kFloat32:
133    case DataType::Type::kFloat64:
134      DCHECK_LE(2u, instruction->GetVectorLength());
135      DCHECK_LE(instruction->GetVectorLength(), 4u);
136      DCHECK(locations->InAt(0).Equals(locations->Out()));  // no code required
137      break;
138    default:
139      LOG(FATAL) << "Unsupported SIMD type";
140      UNREACHABLE();
141  }
142}
143
144// Helper to set up locations for vector unary operations.
145static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation* instruction) {
146  LocationSummary* locations = new (allocator) LocationSummary(instruction);
147  DataType::Type type = instruction->GetPackedType();
148  switch (type) {
149    case DataType::Type::kBool:
150      locations->SetInAt(0, Location::RequiresFpuRegister());
151      locations->SetOut(Location::RequiresFpuRegister(),
152                        instruction->IsVecNot() ? Location::kOutputOverlap
153                                                : Location::kNoOutputOverlap);
154      break;
155    case DataType::Type::kUint8:
156    case DataType::Type::kInt8:
157    case DataType::Type::kUint16:
158    case DataType::Type::kInt16:
159    case DataType::Type::kInt32:
160    case DataType::Type::kInt64:
161    case DataType::Type::kFloat32:
162    case DataType::Type::kFloat64:
163      locations->SetInAt(0, Location::RequiresFpuRegister());
164      locations->SetOut(Location::RequiresFpuRegister(),
165                        (instruction->IsVecNeg() || instruction->IsVecAbs() ||
166                            (instruction->IsVecReduce() && type == DataType::Type::kInt64))
167                            ? Location::kOutputOverlap
168                            : Location::kNoOutputOverlap);
169      break;
170    default:
171      LOG(FATAL) << "Unsupported SIMD type";
172      UNREACHABLE();
173  }
174}
175
176void LocationsBuilderMIPS64::VisitVecReduce(HVecReduce* instruction) {
177  CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
178}
179
180void InstructionCodeGeneratorMIPS64::VisitVecReduce(HVecReduce* instruction) {
181  LocationSummary* locations = instruction->GetLocations();
182  VectorRegister src = VectorRegisterFrom(locations->InAt(0));
183  VectorRegister dst = VectorRegisterFrom(locations->Out());
184  VectorRegister tmp = static_cast<VectorRegister>(FTMP);
185  switch (instruction->GetPackedType()) {
186    case DataType::Type::kInt32:
187      DCHECK_EQ(4u, instruction->GetVectorLength());
188      switch (instruction->GetKind()) {
189        case HVecReduce::kSum:
190          __ Hadd_sD(tmp, src, src);
191          __ IlvlD(dst, tmp, tmp);
192          __ AddvW(dst, dst, tmp);
193          break;
194        case HVecReduce::kMin:
195          __ IlvodW(tmp, src, src);
196          __ Min_sW(tmp, src, tmp);
197          __ IlvlW(dst, tmp, tmp);
198          __ Min_sW(dst, dst, tmp);
199          break;
200        case HVecReduce::kMax:
201          __ IlvodW(tmp, src, src);
202          __ Max_sW(tmp, src, tmp);
203          __ IlvlW(dst, tmp, tmp);
204          __ Max_sW(dst, dst, tmp);
205          break;
206      }
207      break;
208    case DataType::Type::kInt64:
209      DCHECK_EQ(2u, instruction->GetVectorLength());
210      switch (instruction->GetKind()) {
211        case HVecReduce::kSum:
212          __ IlvlD(dst, src, src);
213          __ AddvD(dst, dst, src);
214          break;
215        case HVecReduce::kMin:
216          __ IlvlD(dst, src, src);
217          __ Min_sD(dst, dst, src);
218          break;
219        case HVecReduce::kMax:
220          __ IlvlD(dst, src, src);
221          __ Max_sD(dst, dst, src);
222          break;
223      }
224      break;
225    default:
226      LOG(FATAL) << "Unsupported SIMD type";
227      UNREACHABLE();
228  }
229}
230
231void LocationsBuilderMIPS64::VisitVecCnv(HVecCnv* instruction) {
232  CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
233}
234
235void InstructionCodeGeneratorMIPS64::VisitVecCnv(HVecCnv* instruction) {
236  LocationSummary* locations = instruction->GetLocations();
237  VectorRegister src = VectorRegisterFrom(locations->InAt(0));
238  VectorRegister dst = VectorRegisterFrom(locations->Out());
239  DataType::Type from = instruction->GetInputType();
240  DataType::Type to = instruction->GetResultType();
241  if (from == DataType::Type::kInt32 && to == DataType::Type::kFloat32) {
242    DCHECK_EQ(4u, instruction->GetVectorLength());
243    __ Ffint_sW(dst, src);
244  } else {
245    LOG(FATAL) << "Unsupported SIMD type";
246    UNREACHABLE();
247  }
248}
249
250void LocationsBuilderMIPS64::VisitVecNeg(HVecNeg* instruction) {
251  CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
252}
253
254void InstructionCodeGeneratorMIPS64::VisitVecNeg(HVecNeg* instruction) {
255  LocationSummary* locations = instruction->GetLocations();
256  VectorRegister src = VectorRegisterFrom(locations->InAt(0));
257  VectorRegister dst = VectorRegisterFrom(locations->Out());
258  switch (instruction->GetPackedType()) {
259    case DataType::Type::kUint8:
260    case DataType::Type::kInt8:
261      DCHECK_EQ(16u, instruction->GetVectorLength());
262      __ FillB(dst, ZERO);
263      __ SubvB(dst, dst, src);
264      break;
265    case DataType::Type::kUint16:
266    case DataType::Type::kInt16:
267      DCHECK_EQ(8u, instruction->GetVectorLength());
268      __ FillH(dst, ZERO);
269      __ SubvH(dst, dst, src);
270      break;
271    case DataType::Type::kInt32:
272      DCHECK_EQ(4u, instruction->GetVectorLength());
273      __ FillW(dst, ZERO);
274      __ SubvW(dst, dst, src);
275      break;
276    case DataType::Type::kInt64:
277      DCHECK_EQ(2u, instruction->GetVectorLength());
278      __ FillD(dst, ZERO);
279      __ SubvD(dst, dst, src);
280      break;
281    case DataType::Type::kFloat32:
282      DCHECK_EQ(4u, instruction->GetVectorLength());
283      __ FillW(dst, ZERO);
284      __ FsubW(dst, dst, src);
285      break;
286    case DataType::Type::kFloat64:
287      DCHECK_EQ(2u, instruction->GetVectorLength());
288      __ FillD(dst, ZERO);
289      __ FsubD(dst, dst, src);
290      break;
291    default:
292      LOG(FATAL) << "Unsupported SIMD type";
293      UNREACHABLE();
294  }
295}
296
297void LocationsBuilderMIPS64::VisitVecAbs(HVecAbs* instruction) {
298  CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
299}
300
301void InstructionCodeGeneratorMIPS64::VisitVecAbs(HVecAbs* instruction) {
302  LocationSummary* locations = instruction->GetLocations();
303  VectorRegister src = VectorRegisterFrom(locations->InAt(0));
304  VectorRegister dst = VectorRegisterFrom(locations->Out());
305  switch (instruction->GetPackedType()) {
306    case DataType::Type::kInt8:
307      DCHECK_EQ(16u, instruction->GetVectorLength());
308      __ FillB(dst, ZERO);       // all zeroes
309      __ Add_aB(dst, dst, src);  // dst = abs(0) + abs(src)
310      break;
311    case DataType::Type::kInt16:
312      DCHECK_EQ(8u, instruction->GetVectorLength());
313      __ FillH(dst, ZERO);       // all zeroes
314      __ Add_aH(dst, dst, src);  // dst = abs(0) + abs(src)
315      break;
316    case DataType::Type::kInt32:
317      DCHECK_EQ(4u, instruction->GetVectorLength());
318      __ FillW(dst, ZERO);       // all zeroes
319      __ Add_aW(dst, dst, src);  // dst = abs(0) + abs(src)
320      break;
321    case DataType::Type::kInt64:
322      DCHECK_EQ(2u, instruction->GetVectorLength());
323      __ FillD(dst, ZERO);       // all zeroes
324      __ Add_aD(dst, dst, src);  // dst = abs(0) + abs(src)
325      break;
326    case DataType::Type::kFloat32:
327      DCHECK_EQ(4u, instruction->GetVectorLength());
328      __ LdiW(dst, -1);          // all ones
329      __ SrliW(dst, dst, 1);
330      __ AndV(dst, dst, src);
331      break;
332    case DataType::Type::kFloat64:
333      DCHECK_EQ(2u, instruction->GetVectorLength());
334      __ LdiD(dst, -1);          // all ones
335      __ SrliD(dst, dst, 1);
336      __ AndV(dst, dst, src);
337      break;
338    default:
339      LOG(FATAL) << "Unsupported SIMD type";
340      UNREACHABLE();
341  }
342}
343
344void LocationsBuilderMIPS64::VisitVecNot(HVecNot* instruction) {
345  CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
346}
347
348void InstructionCodeGeneratorMIPS64::VisitVecNot(HVecNot* instruction) {
349  LocationSummary* locations = instruction->GetLocations();
350  VectorRegister src = VectorRegisterFrom(locations->InAt(0));
351  VectorRegister dst = VectorRegisterFrom(locations->Out());
352  switch (instruction->GetPackedType()) {
353    case DataType::Type::kBool:  // special case boolean-not
354      DCHECK_EQ(16u, instruction->GetVectorLength());
355      __ LdiB(dst, 1);
356      __ XorV(dst, dst, src);
357      break;
358    case DataType::Type::kUint8:
359    case DataType::Type::kInt8:
360    case DataType::Type::kUint16:
361    case DataType::Type::kInt16:
362    case DataType::Type::kInt32:
363    case DataType::Type::kInt64:
364    case DataType::Type::kFloat32:
365    case DataType::Type::kFloat64:
366      DCHECK_LE(2u, instruction->GetVectorLength());
367      DCHECK_LE(instruction->GetVectorLength(), 16u);
368      __ NorV(dst, src, src);  // lanes do not matter
369      break;
370    default:
371      LOG(FATAL) << "Unsupported SIMD type";
372      UNREACHABLE();
373  }
374}
375
376// Helper to set up locations for vector binary operations.
377static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
378  LocationSummary* locations = new (allocator) LocationSummary(instruction);
379  switch (instruction->GetPackedType()) {
380    case DataType::Type::kBool:
381    case DataType::Type::kUint8:
382    case DataType::Type::kInt8:
383    case DataType::Type::kUint16:
384    case DataType::Type::kInt16:
385    case DataType::Type::kInt32:
386    case DataType::Type::kInt64:
387    case DataType::Type::kFloat32:
388    case DataType::Type::kFloat64:
389      locations->SetInAt(0, Location::RequiresFpuRegister());
390      locations->SetInAt(1, Location::RequiresFpuRegister());
391      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
392      break;
393    default:
394      LOG(FATAL) << "Unsupported SIMD type";
395      UNREACHABLE();
396  }
397}
398
399void LocationsBuilderMIPS64::VisitVecAdd(HVecAdd* instruction) {
400  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
401}
402
403void InstructionCodeGeneratorMIPS64::VisitVecAdd(HVecAdd* instruction) {
404  LocationSummary* locations = instruction->GetLocations();
405  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
406  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
407  VectorRegister dst = VectorRegisterFrom(locations->Out());
408  switch (instruction->GetPackedType()) {
409    case DataType::Type::kUint8:
410    case DataType::Type::kInt8:
411      DCHECK_EQ(16u, instruction->GetVectorLength());
412      __ AddvB(dst, lhs, rhs);
413      break;
414    case DataType::Type::kUint16:
415    case DataType::Type::kInt16:
416      DCHECK_EQ(8u, instruction->GetVectorLength());
417      __ AddvH(dst, lhs, rhs);
418      break;
419    case DataType::Type::kInt32:
420      DCHECK_EQ(4u, instruction->GetVectorLength());
421      __ AddvW(dst, lhs, rhs);
422      break;
423    case DataType::Type::kInt64:
424      DCHECK_EQ(2u, instruction->GetVectorLength());
425      __ AddvD(dst, lhs, rhs);
426      break;
427    case DataType::Type::kFloat32:
428      DCHECK_EQ(4u, instruction->GetVectorLength());
429      __ FaddW(dst, lhs, rhs);
430      break;
431    case DataType::Type::kFloat64:
432      DCHECK_EQ(2u, instruction->GetVectorLength());
433      __ FaddD(dst, lhs, rhs);
434      break;
435    default:
436      LOG(FATAL) << "Unsupported SIMD type";
437      UNREACHABLE();
438  }
439}
440
441void LocationsBuilderMIPS64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
442  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
443}
444
445void InstructionCodeGeneratorMIPS64::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
446  LocationSummary* locations = instruction->GetLocations();
447  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
448  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
449  VectorRegister dst = VectorRegisterFrom(locations->Out());
450  switch (instruction->GetPackedType()) {
451    case DataType::Type::kUint8:
452      DCHECK_EQ(16u, instruction->GetVectorLength());
453      instruction->IsRounded()
454          ? __ Aver_uB(dst, lhs, rhs)
455          : __ Ave_uB(dst, lhs, rhs);
456      break;
457    case DataType::Type::kInt8:
458      DCHECK_EQ(16u, instruction->GetVectorLength());
459      instruction->IsRounded()
460          ? __ Aver_sB(dst, lhs, rhs)
461          : __ Ave_sB(dst, lhs, rhs);
462      break;
463    case DataType::Type::kUint16:
464      DCHECK_EQ(8u, instruction->GetVectorLength());
465      instruction->IsRounded()
466          ? __ Aver_uH(dst, lhs, rhs)
467          : __ Ave_uH(dst, lhs, rhs);
468      break;
469    case DataType::Type::kInt16:
470      DCHECK_EQ(8u, instruction->GetVectorLength());
471      instruction->IsRounded()
472          ? __ Aver_sH(dst, lhs, rhs)
473          : __ Ave_sH(dst, lhs, rhs);
474      break;
475    default:
476      LOG(FATAL) << "Unsupported SIMD type";
477      UNREACHABLE();
478  }
479}
480
481void LocationsBuilderMIPS64::VisitVecSub(HVecSub* instruction) {
482  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
483}
484
485void InstructionCodeGeneratorMIPS64::VisitVecSub(HVecSub* instruction) {
486  LocationSummary* locations = instruction->GetLocations();
487  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
488  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
489  VectorRegister dst = VectorRegisterFrom(locations->Out());
490  switch (instruction->GetPackedType()) {
491    case DataType::Type::kUint8:
492    case DataType::Type::kInt8:
493      DCHECK_EQ(16u, instruction->GetVectorLength());
494      __ SubvB(dst, lhs, rhs);
495      break;
496    case DataType::Type::kUint16:
497    case DataType::Type::kInt16:
498      DCHECK_EQ(8u, instruction->GetVectorLength());
499      __ SubvH(dst, lhs, rhs);
500      break;
501    case DataType::Type::kInt32:
502      DCHECK_EQ(4u, instruction->GetVectorLength());
503      __ SubvW(dst, lhs, rhs);
504      break;
505    case DataType::Type::kInt64:
506      DCHECK_EQ(2u, instruction->GetVectorLength());
507      __ SubvD(dst, lhs, rhs);
508      break;
509    case DataType::Type::kFloat32:
510      DCHECK_EQ(4u, instruction->GetVectorLength());
511      __ FsubW(dst, lhs, rhs);
512      break;
513    case DataType::Type::kFloat64:
514      DCHECK_EQ(2u, instruction->GetVectorLength());
515      __ FsubD(dst, lhs, rhs);
516      break;
517    default:
518      LOG(FATAL) << "Unsupported SIMD type";
519      UNREACHABLE();
520  }
521}
522
523void LocationsBuilderMIPS64::VisitVecMul(HVecMul* instruction) {
524  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
525}
526
527void InstructionCodeGeneratorMIPS64::VisitVecMul(HVecMul* instruction) {
528  LocationSummary* locations = instruction->GetLocations();
529  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
530  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
531  VectorRegister dst = VectorRegisterFrom(locations->Out());
532  switch (instruction->GetPackedType()) {
533    case DataType::Type::kUint8:
534    case DataType::Type::kInt8:
535      DCHECK_EQ(16u, instruction->GetVectorLength());
536      __ MulvB(dst, lhs, rhs);
537      break;
538    case DataType::Type::kUint16:
539    case DataType::Type::kInt16:
540      DCHECK_EQ(8u, instruction->GetVectorLength());
541      __ MulvH(dst, lhs, rhs);
542      break;
543    case DataType::Type::kInt32:
544      DCHECK_EQ(4u, instruction->GetVectorLength());
545      __ MulvW(dst, lhs, rhs);
546      break;
547    case DataType::Type::kInt64:
548      DCHECK_EQ(2u, instruction->GetVectorLength());
549      __ MulvD(dst, lhs, rhs);
550      break;
551    case DataType::Type::kFloat32:
552      DCHECK_EQ(4u, instruction->GetVectorLength());
553      __ FmulW(dst, lhs, rhs);
554      break;
555    case DataType::Type::kFloat64:
556      DCHECK_EQ(2u, instruction->GetVectorLength());
557      __ FmulD(dst, lhs, rhs);
558      break;
559    default:
560      LOG(FATAL) << "Unsupported SIMD type";
561      UNREACHABLE();
562  }
563}
564
565void LocationsBuilderMIPS64::VisitVecDiv(HVecDiv* instruction) {
566  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
567}
568
569void InstructionCodeGeneratorMIPS64::VisitVecDiv(HVecDiv* instruction) {
570  LocationSummary* locations = instruction->GetLocations();
571  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
572  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
573  VectorRegister dst = VectorRegisterFrom(locations->Out());
574  switch (instruction->GetPackedType()) {
575    case DataType::Type::kFloat32:
576      DCHECK_EQ(4u, instruction->GetVectorLength());
577      __ FdivW(dst, lhs, rhs);
578      break;
579    case DataType::Type::kFloat64:
580      DCHECK_EQ(2u, instruction->GetVectorLength());
581      __ FdivD(dst, lhs, rhs);
582      break;
583    default:
584      LOG(FATAL) << "Unsupported SIMD type";
585      UNREACHABLE();
586  }
587}
588
589void LocationsBuilderMIPS64::VisitVecMin(HVecMin* instruction) {
590  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
591}
592
593void InstructionCodeGeneratorMIPS64::VisitVecMin(HVecMin* instruction) {
594  LocationSummary* locations = instruction->GetLocations();
595  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
596  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
597  VectorRegister dst = VectorRegisterFrom(locations->Out());
598  switch (instruction->GetPackedType()) {
599    case DataType::Type::kUint8:
600      DCHECK_EQ(16u, instruction->GetVectorLength());
601      __ Min_uB(dst, lhs, rhs);
602      break;
603    case DataType::Type::kInt8:
604      DCHECK_EQ(16u, instruction->GetVectorLength());
605      __ Min_sB(dst, lhs, rhs);
606      break;
607    case DataType::Type::kUint16:
608      DCHECK_EQ(8u, instruction->GetVectorLength());
609      __ Min_uH(dst, lhs, rhs);
610      break;
611    case DataType::Type::kInt16:
612      DCHECK_EQ(8u, instruction->GetVectorLength());
613      __ Min_sH(dst, lhs, rhs);
614      break;
615    case DataType::Type::kUint32:
616      DCHECK_EQ(4u, instruction->GetVectorLength());
617      __ Min_uW(dst, lhs, rhs);
618      break;
619    case DataType::Type::kInt32:
620      DCHECK_EQ(4u, instruction->GetVectorLength());
621      __ Min_sW(dst, lhs, rhs);
622      break;
623    case DataType::Type::kUint64:
624      DCHECK_EQ(2u, instruction->GetVectorLength());
625      __ Min_uD(dst, lhs, rhs);
626      break;
627    case DataType::Type::kInt64:
628      DCHECK_EQ(2u, instruction->GetVectorLength());
629      __ Min_sD(dst, lhs, rhs);
630      break;
631    // When one of arguments is NaN, fmin.df returns other argument, but Java expects a NaN value.
632    // TODO: Fix min(x, NaN) cases for float and double.
633    case DataType::Type::kFloat32:
634      DCHECK_EQ(4u, instruction->GetVectorLength());
635      __ FminW(dst, lhs, rhs);
636      break;
637    case DataType::Type::kFloat64:
638      DCHECK_EQ(2u, instruction->GetVectorLength());
639      __ FminD(dst, lhs, rhs);
640      break;
641    default:
642      LOG(FATAL) << "Unsupported SIMD type";
643      UNREACHABLE();
644  }
645}
646
647void LocationsBuilderMIPS64::VisitVecMax(HVecMax* instruction) {
648  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
649}
650
651void InstructionCodeGeneratorMIPS64::VisitVecMax(HVecMax* instruction) {
652  LocationSummary* locations = instruction->GetLocations();
653  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
654  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
655  VectorRegister dst = VectorRegisterFrom(locations->Out());
656  switch (instruction->GetPackedType()) {
657    case DataType::Type::kUint8:
658      DCHECK_EQ(16u, instruction->GetVectorLength());
659      __ Max_uB(dst, lhs, rhs);
660      break;
661    case DataType::Type::kInt8:
662      DCHECK_EQ(16u, instruction->GetVectorLength());
663      __ Max_sB(dst, lhs, rhs);
664      break;
665    case DataType::Type::kUint16:
666      DCHECK_EQ(8u, instruction->GetVectorLength());
667      __ Max_uH(dst, lhs, rhs);
668      break;
669    case DataType::Type::kInt16:
670      DCHECK_EQ(8u, instruction->GetVectorLength());
671      __ Max_sH(dst, lhs, rhs);
672      break;
673    case DataType::Type::kUint32:
674      DCHECK_EQ(4u, instruction->GetVectorLength());
675      __ Max_uW(dst, lhs, rhs);
676      break;
677    case DataType::Type::kInt32:
678      DCHECK_EQ(4u, instruction->GetVectorLength());
679      __ Max_sW(dst, lhs, rhs);
680      break;
681    case DataType::Type::kUint64:
682      DCHECK_EQ(2u, instruction->GetVectorLength());
683      __ Max_uD(dst, lhs, rhs);
684      break;
685    case DataType::Type::kInt64:
686      DCHECK_EQ(2u, instruction->GetVectorLength());
687      __ Max_sD(dst, lhs, rhs);
688      break;
689    // When one of arguments is NaN, fmax.df returns other argument, but Java expects a NaN value.
690    // TODO: Fix max(x, NaN) cases for float and double.
691    case DataType::Type::kFloat32:
692      DCHECK_EQ(4u, instruction->GetVectorLength());
693      __ FmaxW(dst, lhs, rhs);
694      break;
695    case DataType::Type::kFloat64:
696      DCHECK_EQ(2u, instruction->GetVectorLength());
697      __ FmaxD(dst, lhs, rhs);
698      break;
699    default:
700      LOG(FATAL) << "Unsupported SIMD type";
701      UNREACHABLE();
702  }
703}
704
705void LocationsBuilderMIPS64::VisitVecAnd(HVecAnd* instruction) {
706  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
707}
708
709void InstructionCodeGeneratorMIPS64::VisitVecAnd(HVecAnd* instruction) {
710  LocationSummary* locations = instruction->GetLocations();
711  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
712  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
713  VectorRegister dst = VectorRegisterFrom(locations->Out());
714  switch (instruction->GetPackedType()) {
715    case DataType::Type::kBool:
716    case DataType::Type::kUint8:
717    case DataType::Type::kInt8:
718    case DataType::Type::kUint16:
719    case DataType::Type::kInt16:
720    case DataType::Type::kInt32:
721    case DataType::Type::kInt64:
722    case DataType::Type::kFloat32:
723    case DataType::Type::kFloat64:
724      DCHECK_LE(2u, instruction->GetVectorLength());
725      DCHECK_LE(instruction->GetVectorLength(), 16u);
726      __ AndV(dst, lhs, rhs);  // lanes do not matter
727      break;
728    default:
729      LOG(FATAL) << "Unsupported SIMD type";
730      UNREACHABLE();
731  }
732}
733
734void LocationsBuilderMIPS64::VisitVecAndNot(HVecAndNot* instruction) {
735  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
736}
737
738void InstructionCodeGeneratorMIPS64::VisitVecAndNot(HVecAndNot* instruction) {
739  LOG(FATAL) << "No SIMD for " << instruction->GetId();
740}
741
742void LocationsBuilderMIPS64::VisitVecOr(HVecOr* instruction) {
743  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
744}
745
746void InstructionCodeGeneratorMIPS64::VisitVecOr(HVecOr* instruction) {
747  LocationSummary* locations = instruction->GetLocations();
748  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
749  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
750  VectorRegister dst = VectorRegisterFrom(locations->Out());
751  switch (instruction->GetPackedType()) {
752    case DataType::Type::kBool:
753    case DataType::Type::kUint8:
754    case DataType::Type::kInt8:
755    case DataType::Type::kUint16:
756    case DataType::Type::kInt16:
757    case DataType::Type::kInt32:
758    case DataType::Type::kInt64:
759    case DataType::Type::kFloat32:
760    case DataType::Type::kFloat64:
761      DCHECK_LE(2u, instruction->GetVectorLength());
762      DCHECK_LE(instruction->GetVectorLength(), 16u);
763      __ OrV(dst, lhs, rhs);  // lanes do not matter
764      break;
765    default:
766      LOG(FATAL) << "Unsupported SIMD type";
767      UNREACHABLE();
768  }
769}
770
771void LocationsBuilderMIPS64::VisitVecXor(HVecXor* instruction) {
772  CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
773}
774
775void InstructionCodeGeneratorMIPS64::VisitVecXor(HVecXor* instruction) {
776  LocationSummary* locations = instruction->GetLocations();
777  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
778  VectorRegister rhs = VectorRegisterFrom(locations->InAt(1));
779  VectorRegister dst = VectorRegisterFrom(locations->Out());
780  switch (instruction->GetPackedType()) {
781    case DataType::Type::kBool:
782    case DataType::Type::kUint8:
783    case DataType::Type::kInt8:
784    case DataType::Type::kUint16:
785    case DataType::Type::kInt16:
786    case DataType::Type::kInt32:
787    case DataType::Type::kInt64:
788    case DataType::Type::kFloat32:
789    case DataType::Type::kFloat64:
790      DCHECK_LE(2u, instruction->GetVectorLength());
791      DCHECK_LE(instruction->GetVectorLength(), 16u);
792      __ XorV(dst, lhs, rhs);  // lanes do not matter
793      break;
794    default:
795      LOG(FATAL) << "Unsupported SIMD type";
796      UNREACHABLE();
797  }
798}
799
800// Helper to set up locations for vector shift operations.
801static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
802  LocationSummary* locations = new (allocator) LocationSummary(instruction);
803  switch (instruction->GetPackedType()) {
804    case DataType::Type::kUint8:
805    case DataType::Type::kInt8:
806    case DataType::Type::kUint16:
807    case DataType::Type::kInt16:
808    case DataType::Type::kInt32:
809    case DataType::Type::kInt64:
810      locations->SetInAt(0, Location::RequiresFpuRegister());
811      locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
812      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
813      break;
814    default:
815      LOG(FATAL) << "Unsupported SIMD type";
816      UNREACHABLE();
817  }
818}
819
820void LocationsBuilderMIPS64::VisitVecShl(HVecShl* instruction) {
821  CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
822}
823
824void InstructionCodeGeneratorMIPS64::VisitVecShl(HVecShl* instruction) {
825  LocationSummary* locations = instruction->GetLocations();
826  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
827  VectorRegister dst = VectorRegisterFrom(locations->Out());
828  int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
829  switch (instruction->GetPackedType()) {
830    case DataType::Type::kUint8:
831    case DataType::Type::kInt8:
832      DCHECK_EQ(16u, instruction->GetVectorLength());
833      __ SlliB(dst, lhs, value);
834      break;
835    case DataType::Type::kUint16:
836    case DataType::Type::kInt16:
837      DCHECK_EQ(8u, instruction->GetVectorLength());
838      __ SlliH(dst, lhs, value);
839      break;
840    case DataType::Type::kInt32:
841      DCHECK_EQ(4u, instruction->GetVectorLength());
842      __ SlliW(dst, lhs, value);
843      break;
844    case DataType::Type::kInt64:
845      DCHECK_EQ(2u, instruction->GetVectorLength());
846      __ SlliD(dst, lhs, value);
847      break;
848    default:
849      LOG(FATAL) << "Unsupported SIMD type";
850      UNREACHABLE();
851  }
852}
853
854void LocationsBuilderMIPS64::VisitVecShr(HVecShr* instruction) {
855  CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
856}
857
858void InstructionCodeGeneratorMIPS64::VisitVecShr(HVecShr* instruction) {
859  LocationSummary* locations = instruction->GetLocations();
860  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
861  VectorRegister dst = VectorRegisterFrom(locations->Out());
862  int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
863  switch (instruction->GetPackedType()) {
864    case DataType::Type::kUint8:
865    case DataType::Type::kInt8:
866      DCHECK_EQ(16u, instruction->GetVectorLength());
867      __ SraiB(dst, lhs, value);
868      break;
869    case DataType::Type::kUint16:
870    case DataType::Type::kInt16:
871      DCHECK_EQ(8u, instruction->GetVectorLength());
872      __ SraiH(dst, lhs, value);
873      break;
874    case DataType::Type::kInt32:
875      DCHECK_EQ(4u, instruction->GetVectorLength());
876      __ SraiW(dst, lhs, value);
877      break;
878    case DataType::Type::kInt64:
879      DCHECK_EQ(2u, instruction->GetVectorLength());
880      __ SraiD(dst, lhs, value);
881      break;
882    default:
883      LOG(FATAL) << "Unsupported SIMD type";
884      UNREACHABLE();
885  }
886}
887
888void LocationsBuilderMIPS64::VisitVecUShr(HVecUShr* instruction) {
889  CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
890}
891
892void InstructionCodeGeneratorMIPS64::VisitVecUShr(HVecUShr* instruction) {
893  LocationSummary* locations = instruction->GetLocations();
894  VectorRegister lhs = VectorRegisterFrom(locations->InAt(0));
895  VectorRegister dst = VectorRegisterFrom(locations->Out());
896  int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
897  switch (instruction->GetPackedType()) {
898    case DataType::Type::kUint8:
899    case DataType::Type::kInt8:
900      DCHECK_EQ(16u, instruction->GetVectorLength());
901      __ SrliB(dst, lhs, value);
902      break;
903    case DataType::Type::kUint16:
904    case DataType::Type::kInt16:
905      DCHECK_EQ(8u, instruction->GetVectorLength());
906      __ SrliH(dst, lhs, value);
907      break;
908    case DataType::Type::kInt32:
909      DCHECK_EQ(4u, instruction->GetVectorLength());
910      __ SrliW(dst, lhs, value);
911      break;
912    case DataType::Type::kInt64:
913      DCHECK_EQ(2u, instruction->GetVectorLength());
914      __ SrliD(dst, lhs, value);
915      break;
916    default:
917      LOG(FATAL) << "Unsupported SIMD type";
918      UNREACHABLE();
919  }
920}
921
922void LocationsBuilderMIPS64::VisitVecSetScalars(HVecSetScalars* instruction) {
923  LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
924
925  DCHECK_EQ(1u, instruction->InputCount());  // only one input currently implemented
926
927  HInstruction* input = instruction->InputAt(0);
928  bool is_zero = IsZeroBitPattern(input);
929
930  switch (instruction->GetPackedType()) {
931    case DataType::Type::kBool:
932    case DataType::Type::kUint8:
933    case DataType::Type::kInt8:
934    case DataType::Type::kUint16:
935    case DataType::Type::kInt16:
936    case DataType::Type::kInt32:
937    case DataType::Type::kInt64:
938      locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant())
939                                    : Location::RequiresRegister());
940      locations->SetOut(Location::RequiresFpuRegister());
941      break;
942    case DataType::Type::kFloat32:
943    case DataType::Type::kFloat64:
944      locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant())
945                                    : Location::RequiresFpuRegister());
946      locations->SetOut(Location::RequiresFpuRegister());
947      break;
948    default:
949      LOG(FATAL) << "Unsupported SIMD type";
950      UNREACHABLE();
951  }
952}
953
954void InstructionCodeGeneratorMIPS64::VisitVecSetScalars(HVecSetScalars* instruction) {
955  LocationSummary* locations = instruction->GetLocations();
956  VectorRegister dst = VectorRegisterFrom(locations->Out());
957
958  DCHECK_EQ(1u, instruction->InputCount());  // only one input currently implemented
959
960  // Zero out all other elements first.
961  __ FillW(dst, ZERO);
962
963  // Shorthand for any type of zero.
964  if (IsZeroBitPattern(instruction->InputAt(0))) {
965    return;
966  }
967
968  // Set required elements.
969  switch (instruction->GetPackedType()) {
970    case DataType::Type::kBool:
971    case DataType::Type::kUint8:
972    case DataType::Type::kInt8:
973      DCHECK_EQ(16u, instruction->GetVectorLength());
974      __ InsertB(dst, locations->InAt(0).AsRegister<GpuRegister>(), 0);
975      break;
976    case DataType::Type::kUint16:
977    case DataType::Type::kInt16:
978      DCHECK_EQ(8u, instruction->GetVectorLength());
979      __ InsertH(dst, locations->InAt(0).AsRegister<GpuRegister>(), 0);
980      break;
981    case DataType::Type::kInt32:
982      DCHECK_EQ(4u, instruction->GetVectorLength());
983      __ InsertW(dst, locations->InAt(0).AsRegister<GpuRegister>(), 0);
984      break;
985    case DataType::Type::kInt64:
986      DCHECK_EQ(2u, instruction->GetVectorLength());
987      __ InsertD(dst, locations->InAt(0).AsRegister<GpuRegister>(), 0);
988      break;
989    default:
990      LOG(FATAL) << "Unsupported SIMD type";
991      UNREACHABLE();
992  }
993}
994
995// Helper to set up locations for vector accumulations.
996static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* instruction) {
997  LocationSummary* locations = new (allocator) LocationSummary(instruction);
998  switch (instruction->GetPackedType()) {
999    case DataType::Type::kUint8:
1000    case DataType::Type::kInt8:
1001    case DataType::Type::kUint16:
1002    case DataType::Type::kInt16:
1003    case DataType::Type::kInt32:
1004    case DataType::Type::kInt64:
1005      locations->SetInAt(0, Location::RequiresFpuRegister());
1006      locations->SetInAt(1, Location::RequiresFpuRegister());
1007      locations->SetInAt(2, Location::RequiresFpuRegister());
1008      locations->SetOut(Location::SameAsFirstInput());
1009      break;
1010    default:
1011      LOG(FATAL) << "Unsupported SIMD type";
1012      UNREACHABLE();
1013  }
1014}
1015
1016void LocationsBuilderMIPS64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
1017  CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
1018}
1019
1020void InstructionCodeGeneratorMIPS64::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
1021  LocationSummary* locations = instruction->GetLocations();
1022  VectorRegister acc = VectorRegisterFrom(locations->InAt(0));
1023  VectorRegister left = VectorRegisterFrom(locations->InAt(1));
1024  VectorRegister right = VectorRegisterFrom(locations->InAt(2));
1025  switch (instruction->GetPackedType()) {
1026    case DataType::Type::kUint8:
1027    case DataType::Type::kInt8:
1028      DCHECK_EQ(16u, instruction->GetVectorLength());
1029      if (instruction->GetOpKind() == HInstruction::kAdd) {
1030        __ MaddvB(acc, left, right);
1031      } else {
1032        __ MsubvB(acc, left, right);
1033      }
1034      break;
1035    case DataType::Type::kUint16:
1036    case DataType::Type::kInt16:
1037      DCHECK_EQ(8u, instruction->GetVectorLength());
1038      if (instruction->GetOpKind() == HInstruction::kAdd) {
1039        __ MaddvH(acc, left, right);
1040      } else {
1041        __ MsubvH(acc, left, right);
1042      }
1043      break;
1044    case DataType::Type::kInt32:
1045      DCHECK_EQ(4u, instruction->GetVectorLength());
1046      if (instruction->GetOpKind() == HInstruction::kAdd) {
1047        __ MaddvW(acc, left, right);
1048      } else {
1049        __ MsubvW(acc, left, right);
1050      }
1051      break;
1052    case DataType::Type::kInt64:
1053      DCHECK_EQ(2u, instruction->GetVectorLength());
1054      if (instruction->GetOpKind() == HInstruction::kAdd) {
1055        __ MaddvD(acc, left, right);
1056      } else {
1057        __ MsubvD(acc, left, right);
1058      }
1059      break;
1060    default:
1061      LOG(FATAL) << "Unsupported SIMD type";
1062      UNREACHABLE();
1063  }
1064}
1065
1066void LocationsBuilderMIPS64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
1067  CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
1068  LocationSummary* locations = instruction->GetLocations();
1069  // All conversions require at least one temporary register.
1070  locations->AddTemp(Location::RequiresFpuRegister());
1071  // Some conversions require a second temporary register.
1072  HVecOperation* a = instruction->InputAt(1)->AsVecOperation();
1073  HVecOperation* b = instruction->InputAt(2)->AsVecOperation();
1074  DCHECK_EQ(HVecOperation::ToSignedType(a->GetPackedType()),
1075            HVecOperation::ToSignedType(b->GetPackedType()));
1076  switch (a->GetPackedType()) {
1077    case DataType::Type::kInt32:
1078      if (instruction->GetPackedType() == DataType::Type::kInt32) {
1079        break;
1080      }
1081      FALLTHROUGH_INTENDED;
1082    case DataType::Type::kUint8:
1083    case DataType::Type::kInt8:
1084    case DataType::Type::kUint16:
1085    case DataType::Type::kInt16:
1086      locations->AddTemp(Location::RequiresFpuRegister());
1087      break;
1088    default:
1089      break;
1090  }
1091}
1092
1093void InstructionCodeGeneratorMIPS64::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
1094  LocationSummary* locations = instruction->GetLocations();
1095  VectorRegister acc = VectorRegisterFrom(locations->InAt(0));
1096  VectorRegister left = VectorRegisterFrom(locations->InAt(1));
1097  VectorRegister right = VectorRegisterFrom(locations->InAt(2));
1098  VectorRegister tmp = static_cast<VectorRegister>(FTMP);
1099  VectorRegister tmp1 = VectorRegisterFrom(locations->GetTemp(0));
1100
1101  DCHECK(locations->InAt(0).Equals(locations->Out()));
1102
1103  // Handle all feasible acc_T += sad(a_S, b_S) type combinations (T x S).
1104  HVecOperation* a = instruction->InputAt(1)->AsVecOperation();
1105  HVecOperation* b = instruction->InputAt(2)->AsVecOperation();
1106  DCHECK_EQ(HVecOperation::ToSignedType(a->GetPackedType()),
1107            HVecOperation::ToSignedType(b->GetPackedType()));
1108  switch (a->GetPackedType()) {
1109    case DataType::Type::kUint8:
1110    case DataType::Type::kInt8:
1111      DCHECK_EQ(16u, a->GetVectorLength());
1112      switch (instruction->GetPackedType()) {
1113        case DataType::Type::kUint16:
1114        case DataType::Type::kInt16: {
1115          DCHECK_EQ(8u, instruction->GetVectorLength());
1116          VectorRegister tmp2 = VectorRegisterFrom(locations->GetTemp(1));
1117          __ FillB(tmp, ZERO);
1118          __ Hadd_sH(tmp1, left, tmp);
1119          __ Hadd_sH(tmp2, right, tmp);
1120          __ Asub_sH(tmp1, tmp1, tmp2);
1121          __ AddvH(acc, acc, tmp1);
1122          __ Hadd_sH(tmp1, tmp, left);
1123          __ Hadd_sH(tmp2, tmp, right);
1124          __ Asub_sH(tmp1, tmp1, tmp2);
1125          __ AddvH(acc, acc, tmp1);
1126          break;
1127        }
1128        case DataType::Type::kInt32: {
1129          DCHECK_EQ(4u, instruction->GetVectorLength());
1130          VectorRegister tmp2 = VectorRegisterFrom(locations->GetTemp(1));
1131          __ FillB(tmp, ZERO);
1132          __ Hadd_sH(tmp1, left, tmp);
1133          __ Hadd_sH(tmp2, right, tmp);
1134          __ Asub_sH(tmp1, tmp1, tmp2);
1135          __ Hadd_sW(tmp1, tmp1, tmp1);
1136          __ AddvW(acc, acc, tmp1);
1137          __ Hadd_sH(tmp1, tmp, left);
1138          __ Hadd_sH(tmp2, tmp, right);
1139          __ Asub_sH(tmp1, tmp1, tmp2);
1140          __ Hadd_sW(tmp1, tmp1, tmp1);
1141          __ AddvW(acc, acc, tmp1);
1142          break;
1143        }
1144        case DataType::Type::kInt64: {
1145          DCHECK_EQ(2u, instruction->GetVectorLength());
1146          VectorRegister tmp2 = VectorRegisterFrom(locations->GetTemp(1));
1147          __ FillB(tmp, ZERO);
1148          __ Hadd_sH(tmp1, left, tmp);
1149          __ Hadd_sH(tmp2, right, tmp);
1150          __ Asub_sH(tmp1, tmp1, tmp2);
1151          __ Hadd_sW(tmp1, tmp1, tmp1);
1152          __ Hadd_sD(tmp1, tmp1, tmp1);
1153          __ AddvD(acc, acc, tmp1);
1154          __ Hadd_sH(tmp1, tmp, left);
1155          __ Hadd_sH(tmp2, tmp, right);
1156          __ Asub_sH(tmp1, tmp1, tmp2);
1157          __ Hadd_sW(tmp1, tmp1, tmp1);
1158          __ Hadd_sD(tmp1, tmp1, tmp1);
1159          __ AddvD(acc, acc, tmp1);
1160          break;
1161        }
1162        default:
1163          LOG(FATAL) << "Unsupported SIMD type";
1164          UNREACHABLE();
1165      }
1166      break;
1167    case DataType::Type::kUint16:
1168    case DataType::Type::kInt16:
1169      DCHECK_EQ(8u, a->GetVectorLength());
1170      switch (instruction->GetPackedType()) {
1171        case DataType::Type::kInt32: {
1172          DCHECK_EQ(4u, instruction->GetVectorLength());
1173          VectorRegister tmp2 = VectorRegisterFrom(locations->GetTemp(1));
1174          __ FillH(tmp, ZERO);
1175          __ Hadd_sW(tmp1, left, tmp);
1176          __ Hadd_sW(tmp2, right, tmp);
1177          __ Asub_sW(tmp1, tmp1, tmp2);
1178          __ AddvW(acc, acc, tmp1);
1179          __ Hadd_sW(tmp1, tmp, left);
1180          __ Hadd_sW(tmp2, tmp, right);
1181          __ Asub_sW(tmp1, tmp1, tmp2);
1182          __ AddvW(acc, acc, tmp1);
1183          break;
1184        }
1185        case DataType::Type::kInt64: {
1186          DCHECK_EQ(2u, instruction->GetVectorLength());
1187          VectorRegister tmp2 = VectorRegisterFrom(locations->GetTemp(1));
1188          __ FillH(tmp, ZERO);
1189          __ Hadd_sW(tmp1, left, tmp);
1190          __ Hadd_sW(tmp2, right, tmp);
1191          __ Asub_sW(tmp1, tmp1, tmp2);
1192          __ Hadd_sD(tmp1, tmp1, tmp1);
1193          __ AddvD(acc, acc, tmp1);
1194          __ Hadd_sW(tmp1, tmp, left);
1195          __ Hadd_sW(tmp2, tmp, right);
1196          __ Asub_sW(tmp1, tmp1, tmp2);
1197          __ Hadd_sD(tmp1, tmp1, tmp1);
1198          __ AddvD(acc, acc, tmp1);
1199          break;
1200        }
1201        default:
1202          LOG(FATAL) << "Unsupported SIMD type";
1203          UNREACHABLE();
1204      }
1205      break;
1206    case DataType::Type::kInt32:
1207      DCHECK_EQ(4u, a->GetVectorLength());
1208      switch (instruction->GetPackedType()) {
1209        case DataType::Type::kInt32: {
1210          DCHECK_EQ(4u, instruction->GetVectorLength());
1211          __ FillW(tmp, ZERO);
1212          __ SubvW(tmp1, left, right);
1213          __ Add_aW(tmp1, tmp1, tmp);
1214          __ AddvW(acc, acc, tmp1);
1215          break;
1216        }
1217        case DataType::Type::kInt64: {
1218          DCHECK_EQ(2u, instruction->GetVectorLength());
1219          VectorRegister tmp2 = VectorRegisterFrom(locations->GetTemp(1));
1220          __ FillW(tmp, ZERO);
1221          __ Hadd_sD(tmp1, left, tmp);
1222          __ Hadd_sD(tmp2, right, tmp);
1223          __ Asub_sD(tmp1, tmp1, tmp2);
1224          __ AddvD(acc, acc, tmp1);
1225          __ Hadd_sD(tmp1, tmp, left);
1226          __ Hadd_sD(tmp2, tmp, right);
1227          __ Asub_sD(tmp1, tmp1, tmp2);
1228          __ AddvD(acc, acc, tmp1);
1229          break;
1230        }
1231        default:
1232          LOG(FATAL) << "Unsupported SIMD type";
1233          UNREACHABLE();
1234      }
1235      break;
1236    case DataType::Type::kInt64: {
1237      DCHECK_EQ(2u, a->GetVectorLength());
1238      switch (instruction->GetPackedType()) {
1239        case DataType::Type::kInt64: {
1240          DCHECK_EQ(2u, instruction->GetVectorLength());
1241          __ FillD(tmp, ZERO);
1242          __ SubvD(tmp1, left, right);
1243          __ Add_aD(tmp1, tmp1, tmp);
1244          __ AddvD(acc, acc, tmp1);
1245          break;
1246        }
1247        default:
1248          LOG(FATAL) << "Unsupported SIMD type";
1249          UNREACHABLE();
1250      }
1251      break;
1252    }
1253    default:
1254      LOG(FATAL) << "Unsupported SIMD type";
1255      UNREACHABLE();
1256  }
1257}
1258
1259// Helper to set up locations for vector memory operations.
1260static void CreateVecMemLocations(ArenaAllocator* allocator,
1261                                  HVecMemoryOperation* instruction,
1262                                  bool is_load) {
1263  LocationSummary* locations = new (allocator) LocationSummary(instruction);
1264  switch (instruction->GetPackedType()) {
1265    case DataType::Type::kBool:
1266    case DataType::Type::kUint8:
1267    case DataType::Type::kInt8:
1268    case DataType::Type::kUint16:
1269    case DataType::Type::kInt16:
1270    case DataType::Type::kInt32:
1271    case DataType::Type::kInt64:
1272    case DataType::Type::kFloat32:
1273    case DataType::Type::kFloat64:
1274      locations->SetInAt(0, Location::RequiresRegister());
1275      locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1276      if (is_load) {
1277        locations->SetOut(Location::RequiresFpuRegister());
1278      } else {
1279        locations->SetInAt(2, Location::RequiresFpuRegister());
1280      }
1281      break;
1282    default:
1283      LOG(FATAL) << "Unsupported SIMD type";
1284      UNREACHABLE();
1285  }
1286}
1287
1288// Helper to prepare register and offset for vector memory operations. Returns the offset and sets
1289// the output parameter adjusted_base to the original base or to a reserved temporary register (AT).
1290int32_t InstructionCodeGeneratorMIPS64::VecAddress(LocationSummary* locations,
1291                                                   size_t size,
1292                                                   /* out */ GpuRegister* adjusted_base) {
1293  GpuRegister base = locations->InAt(0).AsRegister<GpuRegister>();
1294  Location index = locations->InAt(1);
1295  int scale = TIMES_1;
1296  switch (size) {
1297    case 2: scale = TIMES_2; break;
1298    case 4: scale = TIMES_4; break;
1299    case 8: scale = TIMES_8; break;
1300    default: break;
1301  }
1302  int32_t offset = mirror::Array::DataOffset(size).Int32Value();
1303
1304  if (index.IsConstant()) {
1305    offset += index.GetConstant()->AsIntConstant()->GetValue() << scale;
1306    __ AdjustBaseOffsetAndElementSizeShift(base, offset, scale);
1307    *adjusted_base = base;
1308  } else {
1309    GpuRegister index_reg = index.AsRegister<GpuRegister>();
1310    if (scale != TIMES_1) {
1311      __ Dlsa(AT, index_reg, base, scale);
1312    } else {
1313      __ Daddu(AT, base, index_reg);
1314    }
1315    *adjusted_base = AT;
1316  }
1317  return offset;
1318}
1319
1320void LocationsBuilderMIPS64::VisitVecLoad(HVecLoad* instruction) {
1321  CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /* is_load */ true);
1322}
1323
1324void InstructionCodeGeneratorMIPS64::VisitVecLoad(HVecLoad* instruction) {
1325  LocationSummary* locations = instruction->GetLocations();
1326  size_t size = DataType::Size(instruction->GetPackedType());
1327  VectorRegister reg = VectorRegisterFrom(locations->Out());
1328  GpuRegister base;
1329  int32_t offset = VecAddress(locations, size, &base);
1330  switch (instruction->GetPackedType()) {
1331    case DataType::Type::kBool:
1332    case DataType::Type::kUint8:
1333    case DataType::Type::kInt8:
1334      DCHECK_EQ(16u, instruction->GetVectorLength());
1335      __ LdB(reg, base, offset);
1336      break;
1337    case DataType::Type::kUint16:
1338    case DataType::Type::kInt16:
1339      // Loading 8-bytes (needed if dealing with compressed strings in StringCharAt) from unaligned
1340      // memory address may cause a trap to the kernel if the CPU doesn't directly support unaligned
1341      // loads and stores.
1342      // TODO: Implement support for StringCharAt.
1343      DCHECK(!instruction->IsStringCharAt());
1344      DCHECK_EQ(8u, instruction->GetVectorLength());
1345      __ LdH(reg, base, offset);
1346      break;
1347    case DataType::Type::kInt32:
1348    case DataType::Type::kFloat32:
1349      DCHECK_EQ(4u, instruction->GetVectorLength());
1350      __ LdW(reg, base, offset);
1351      break;
1352    case DataType::Type::kInt64:
1353    case DataType::Type::kFloat64:
1354      DCHECK_EQ(2u, instruction->GetVectorLength());
1355      __ LdD(reg, base, offset);
1356      break;
1357    default:
1358      LOG(FATAL) << "Unsupported SIMD type";
1359      UNREACHABLE();
1360  }
1361}
1362
1363void LocationsBuilderMIPS64::VisitVecStore(HVecStore* instruction) {
1364  CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /* is_load */ false);
1365}
1366
1367void InstructionCodeGeneratorMIPS64::VisitVecStore(HVecStore* instruction) {
1368  LocationSummary* locations = instruction->GetLocations();
1369  size_t size = DataType::Size(instruction->GetPackedType());
1370  VectorRegister reg = VectorRegisterFrom(locations->InAt(2));
1371  GpuRegister base;
1372  int32_t offset = VecAddress(locations, size, &base);
1373  switch (instruction->GetPackedType()) {
1374    case DataType::Type::kBool:
1375    case DataType::Type::kUint8:
1376    case DataType::Type::kInt8:
1377      DCHECK_EQ(16u, instruction->GetVectorLength());
1378      __ StB(reg, base, offset);
1379      break;
1380    case DataType::Type::kUint16:
1381    case DataType::Type::kInt16:
1382      DCHECK_EQ(8u, instruction->GetVectorLength());
1383      __ StH(reg, base, offset);
1384      break;
1385    case DataType::Type::kInt32:
1386    case DataType::Type::kFloat32:
1387      DCHECK_EQ(4u, instruction->GetVectorLength());
1388      __ StW(reg, base, offset);
1389      break;
1390    case DataType::Type::kInt64:
1391    case DataType::Type::kFloat64:
1392      DCHECK_EQ(2u, instruction->GetVectorLength());
1393      __ StD(reg, base, offset);
1394      break;
1395    default:
1396      LOG(FATAL) << "Unsupported SIMD type";
1397      UNREACHABLE();
1398  }
1399}
1400
1401#undef __
1402
1403}  // namespace mips64
1404}  // namespace art
1405