1020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/* 2020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * Copyright 2012 The Android Open Source Project 3020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * 4020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * Use of this source code is governed by a BSD-style license that can be 5020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * found in the LICENSE file. 6020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com */ 7020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 8ab562b317946d28e706430886058b8bb951dbae1djsollen@google.com#include "SkBlitRow_opts_arm_neon.h" 9020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 10020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkBlitMask.h" 11020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkBlitRow.h" 12020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkColorPriv.h" 13020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkDither.h" 14020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkMathPriv.h" 15020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkUtils.h" 16020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 17020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include "SkCachePreload_arm.h" 18020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 19020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#include <arm_neon.h> 20020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 21020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32A_D565_Opaque_neon(uint16_t* SK_RESTRICT dst, 22020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const SkPMColor* SK_RESTRICT src, int count, 23020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com U8CPU alpha, int /*x*/, int /*y*/) { 24020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkASSERT(255 == alpha); 25020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 26020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count >= 8) { 27d8f6ff71cd1c36a388eec93282a1fd1c0877a7f5djsollen@google.com uint16_t* SK_RESTRICT keep_dst = 0; 28935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 29020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com asm volatile ( 30020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "ands ip, %[count], #7 \n\t" 31020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.u8 d31, #1<<7 \n\t" 32020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.16 {q12}, [%[dst]] \n\t" 33020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld4.8 {d0-d3}, [%[src]] \n\t" 34020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // Thumb does not support the standard ARM conditional 35020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // instructions but instead requires the 'it' instruction 36020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // to signal conditional execution 37020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "it eq \n\t" 38020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "moveq ip, #8 \n\t" 39020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "mov %[keep_dst], %[dst] \n\t" 40935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 41020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "add %[src], %[src], ip, LSL#2 \n\t" 42020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "add %[dst], %[dst], ip, LSL#1 \n\t" 43020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "subs %[count], %[count], ip \n\t" 44020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "b 9f \n\t" 45020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // LOOP 46020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "2: \n\t" 47935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 48020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.16 {q12}, [%[dst]]! \n\t" 49020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld4.8 {d0-d3}, [%[src]]! \n\t" 50020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.16 {q10}, [%[keep_dst]] \n\t" 51020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "sub %[keep_dst], %[dst], #8*2 \n\t" 52020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "subs %[count], %[count], #8 \n\t" 53020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "9: \n\t" 54020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "pld [%[dst],#32] \n\t" 55020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // expand 0565 q12 to 8888 {d4-d7} 56020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.u16 d4, q12 \n\t" 57020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q11, q12, #5 \n\t" 58020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q10, q12, #6+5 \n\t" 59020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.u16 d5, q11 \n\t" 60020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.u16 d6, q10 \n\t" 61020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u8 d4, d4, #3 \n\t" 62020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u8 d5, d5, #2 \n\t" 63020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u8 d6, d6, #3 \n\t" 64935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 65020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q14, d31 \n\t" 66020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q13, d31 \n\t" 67020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q12, d31 \n\t" 68935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 69020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // duplicate in 4/2/1 & 8pix vsns 70020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmvn.8 d30, d3 \n\t" 71020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmlal.u8 q14, d30, d6 \n\t" 72020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmlal.u8 q13, d30, d5 \n\t" 73020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmlal.u8 q12, d30, d4 \n\t" 74020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q8, q14, #5 \n\t" 75020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q9, q13, #6 \n\t" 76020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddhn.u16 d6, q14, q8 \n\t" 77020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q8, q12, #5 \n\t" 78020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddhn.u16 d5, q13, q9 \n\t" 79020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vqadd.u8 d6, d6, d0 \n\t" // moved up 80020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddhn.u16 d4, q12, q8 \n\t" 81020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // intentionally don't calculate alpha 82020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // result in d4-d6 83935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 84020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vqadd.u8 d5, d5, d1 \n\t" 85020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vqadd.u8 d4, d4, d2 \n\t" 86935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 87020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // pack 8888 {d4-d6} to 0565 q10 88020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshll.u8 q10, d6, #8 \n\t" 89020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshll.u8 q3, d5, #8 \n\t" 90020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshll.u8 q2, d4, #8 \n\t" 91020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsri.u16 q10, q3, #5 \n\t" 92020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsri.u16 q10, q2, #11 \n\t" 93935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 94020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "bne 2b \n\t" 95935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 96020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "1: \n\t" 97020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.16 {q10}, [%[keep_dst]] \n\t" 98020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [count] "+r" (count) 99935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com : [dst] "r" (dst), [keep_dst] "r" (keep_dst), [src] "r" (src) 100020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : "ip", "cc", "memory", "d0","d1","d2","d3","d4","d5","d6","d7", 101020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29", 102020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "d30","d31" 103020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 104020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 105935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com else 106020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com { // handle count < 8 107d8f6ff71cd1c36a388eec93282a1fd1c0877a7f5djsollen@google.com uint16_t* SK_RESTRICT keep_dst = 0; 108935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 109020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com asm volatile ( 110020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.u8 d31, #1<<7 \n\t" 111020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "mov %[keep_dst], %[dst] \n\t" 112935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 113020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "tst %[count], #4 \n\t" 114020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 14f \n\t" 115020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.16 {d25}, [%[dst]]! \n\t" 116020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.32 {q1}, [%[src]]! \n\t" 117935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 118020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "14: \n\t" 119020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "tst %[count], #2 \n\t" 120020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 12f \n\t" 121020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.32 {d24[1]}, [%[dst]]! \n\t" 122020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.32 {d1}, [%[src]]! \n\t" 123935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 124020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "12: \n\t" 125020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "tst %[count], #1 \n\t" 126020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 11f \n\t" 127020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.16 {d24[1]}, [%[dst]]! \n\t" 128020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.32 {d0[1]}, [%[src]]! \n\t" 129935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 130020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "11: \n\t" 131020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // unzips achieve the same as a vld4 operation 132020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vuzpq.u16 q0, q1 \n\t" 133020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vuzp.u8 d0, d1 \n\t" 134020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vuzp.u8 d2, d3 \n\t" 135020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // expand 0565 q12 to 8888 {d4-d7} 136020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.u16 d4, q12 \n\t" 137020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q11, q12, #5 \n\t" 138020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q10, q12, #6+5 \n\t" 139020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.u16 d5, q11 \n\t" 140020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.u16 d6, q10 \n\t" 141020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u8 d4, d4, #3 \n\t" 142020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u8 d5, d5, #2 \n\t" 143020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u8 d6, d6, #3 \n\t" 144935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 145020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q14, d31 \n\t" 146020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q13, d31 \n\t" 147020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q12, d31 \n\t" 148935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 149020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // duplicate in 4/2/1 & 8pix vsns 150020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmvn.8 d30, d3 \n\t" 151020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmlal.u8 q14, d30, d6 \n\t" 152020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmlal.u8 q13, d30, d5 \n\t" 153020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmlal.u8 q12, d30, d4 \n\t" 154020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q8, q14, #5 \n\t" 155020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q9, q13, #6 \n\t" 156020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddhn.u16 d6, q14, q8 \n\t" 157020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q8, q12, #5 \n\t" 158020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddhn.u16 d5, q13, q9 \n\t" 159020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vqadd.u8 d6, d6, d0 \n\t" // moved up 160020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddhn.u16 d4, q12, q8 \n\t" 161020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // intentionally don't calculate alpha 162020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // result in d4-d6 163935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 164020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vqadd.u8 d5, d5, d1 \n\t" 165020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vqadd.u8 d4, d4, d2 \n\t" 166935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 167020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // pack 8888 {d4-d6} to 0565 q10 168020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshll.u8 q10, d6, #8 \n\t" 169020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshll.u8 q3, d5, #8 \n\t" 170020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshll.u8 q2, d4, #8 \n\t" 171020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsri.u16 q10, q3, #5 \n\t" 172020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsri.u16 q10, q2, #11 \n\t" 173935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 174020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // store 175020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "tst %[count], #4 \n\t" 176020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 24f \n\t" 177020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.16 {d21}, [%[keep_dst]]! \n\t" 178935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 179020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "24: \n\t" 180020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "tst %[count], #2 \n\t" 181020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 22f \n\t" 182020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.32 {d20[1]}, [%[keep_dst]]! \n\t" 183935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 184020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "22: \n\t" 185020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "tst %[count], #1 \n\t" 186020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 21f \n\t" 187020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.16 {d20[1]}, [%[keep_dst]]! \n\t" 188935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 189020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "21: \n\t" 190020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [count] "+r" (count) 191020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [dst] "r" (dst), [keep_dst] "r" (keep_dst), [src] "r" (src) 192020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : "ip", "cc", "memory", "d0","d1","d2","d3","d4","d5","d6","d7", 193020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29", 194020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "d30","d31" 195020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 196020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 197020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 198020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 199020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32A_D565_Blend_neon(uint16_t* SK_RESTRICT dst, 200020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const SkPMColor* SK_RESTRICT src, int count, 201020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com U8CPU alpha, int /*x*/, int /*y*/) { 202020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 203020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com U8CPU alpha_for_asm = alpha; 204020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 205020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com asm volatile ( 206020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com /* This code implements a Neon version of S32A_D565_Blend. The output differs from 207020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * the original in two respects: 208020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * 1. The results have a few mismatches compared to the original code. These mismatches 209020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * never exceed 1. It's possible to improve accuracy vs. a floating point 210020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * implementation by introducing rounding right shifts (vrshr) for the final stage. 211020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * Rounding is not present in the code below, because although results would be closer 212935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * to a floating point implementation, the number of mismatches compared to the 213020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * original code would be far greater. 214020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * 2. On certain inputs, the original code can overflow, causing colour channels to 215020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * mix. Although the Neon code can also overflow, it doesn't allow one colour channel 216020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * to affect another. 217020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com */ 218935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 219020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#if 1 220935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* reflects SkAlpha255To256()'s change from a+a>>7 to a+1 */ 221020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "add %[alpha], %[alpha], #1 \n\t" // adjust range of alpha 0-256 222020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#else 223020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "add %[alpha], %[alpha], %[alpha], lsr #7 \n\t" // adjust range of alpha 0-256 224020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 225020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.u16 q3, #255 \n\t" // set up constant 226020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "movs r4, %[count], lsr #3 \n\t" // calc. count>>3 227020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.u16 d2[0], %[alpha] \n\t" // move alpha to Neon 228020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "beq 2f \n\t" // if count8 == 0, exit 229020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.u16 q15, #0x1f \n\t" // set up blue mask 230935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 231020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "1: \n\t" 232020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.u16 {d0, d1}, [%[dst]] \n\t" // load eight dst RGB565 pixels 233020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "subs r4, r4, #1 \n\t" // decrement loop counter 234020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld4.u8 {d24, d25, d26, d27}, [%[src]]! \n\t" // load eight src ABGR32 pixels 235020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // and deinterleave 236935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 237020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshl.u16 q9, q0, #5 \n\t" // shift green to top of lanes 238020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vand q10, q0, q15 \n\t" // extract blue 239020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q8, q0, #11 \n\t" // extract red 240020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q9, q9, #10 \n\t" // extract green 241020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // dstrgb = {q8, q9, q10} 242935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 243020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d24, d24, #3 \n\t" // shift red to 565 range 244020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d25, d25, #2 \n\t" // shift green to 565 range 245020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d26, d26, #3 \n\t" // shift blue to 565 range 246935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 247020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q11, d24 \n\t" // widen red to 16 bits 248020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q12, d25 \n\t" // widen green to 16 bits 249020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q14, d27 \n\t" // widen alpha to 16 bits 250020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovl.u8 q13, d26 \n\t" // widen blue to 16 bits 251020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // srcrgba = {q11, q12, q13, q14} 252935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 253020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.u16 q2, q14, d2[0] \n\t" // sa * src_scale 254020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.u16 q11, q11, d2[0] \n\t" // red result = src_red * src_scale 255020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.u16 q12, q12, d2[0] \n\t" // grn result = src_grn * src_scale 256020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.u16 q13, q13, d2[0] \n\t" // blu result = src_blu * src_scale 257935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 258020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q2, q2, #8 \n\t" // sa * src_scale >> 8 259020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsub.u16 q2, q3, q2 \n\t" // 255 - (sa * src_scale >> 8) 260020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // dst_scale = q2 261935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 262020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmla.u16 q11, q8, q2 \n\t" // red result += dst_red * dst_scale 263020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmla.u16 q12, q9, q2 \n\t" // grn result += dst_grn * dst_scale 264020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmla.u16 q13, q10, q2 \n\t" // blu result += dst_blu * dst_scale 265020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 266020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#if 1 267935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com // trying for a better match with SkDiv255Round(a) 268935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com // C alg is: a+=128; (a+a>>8)>>8 269935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com // we'll use just a rounding shift [q2 is available for scratch] 270020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vrshr.u16 q11, q11, #8 \n\t" // shift down red 271020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vrshr.u16 q12, q12, #8 \n\t" // shift down green 272020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vrshr.u16 q13, q13, #8 \n\t" // shift down blue 273020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#else 274935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com // arm's original "truncating divide by 256" 275020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q11, q11, #8 \n\t" // shift down red 276020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q12, q12, #8 \n\t" // shift down green 277020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q13, q13, #8 \n\t" // shift down blue 278020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 279935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 280020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsli.u16 q13, q12, #5 \n\t" // insert green into blue 281020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsli.u16 q13, q11, #11 \n\t" // insert red into green/blue 282020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.16 {d26, d27}, [%[dst]]! \n\t" // write pixel back to dst, update ptr 283935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 284020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "bne 1b \n\t" // if counter != 0, loop 285020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "2: \n\t" // exit 286935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 287020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [src] "+r" (src), [dst] "+r" (dst), [count] "+r" (count), [alpha] "+r" (alpha_for_asm) 288020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : 289020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : "cc", "memory", "r4", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" 290020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 291020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 292020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com count &= 7; 293020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count > 0) { 294020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com do { 295020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColor sc = *src++; 296020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (sc) { 297020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint16_t dc = *dst; 298020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned dst_scale = 255 - SkMulDiv255Round(SkGetPackedA32(sc), alpha); 299020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned dr = SkMulS16(SkPacked32ToR16(sc), alpha) + SkMulS16(SkGetPackedR16(dc), dst_scale); 300020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned dg = SkMulS16(SkPacked32ToG16(sc), alpha) + SkMulS16(SkGetPackedG16(dc), dst_scale); 301020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned db = SkMulS16(SkPacked32ToB16(sc), alpha) + SkMulS16(SkGetPackedB16(dc), dst_scale); 302020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst = SkPackRGB16(SkDiv255Round(dr), SkDiv255Round(dg), SkDiv255Round(db)); 303020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 304020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst += 1; 305020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } while (--count != 0); 306020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 307020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 308020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 309020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/* dither matrix for Neon, derived from gDitherMatrix_3Bit_16. 310020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * each dither value is spaced out into byte lanes, and repeated 311020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * to allow an 8-byte load from offsets 0, 1, 2 or 3 from the 312020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * start of each row. 313020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com */ 314020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comstatic const uint8_t gDitherMatrix_Neon[48] = { 315020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 0, 4, 1, 5, 0, 4, 1, 5, 0, 4, 1, 5, 316020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 6, 2, 7, 3, 6, 2, 7, 3, 6, 2, 7, 3, 317020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1, 5, 0, 4, 1, 5, 0, 4, 1, 5, 0, 4, 318020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 7, 3, 6, 2, 7, 3, 6, 2, 7, 3, 6, 2, 319935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 320020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com}; 321020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 322020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32_D565_Blend_Dither_neon(uint16_t *dst, const SkPMColor *src, 323020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int count, U8CPU alpha, int x, int y) 324020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com{ 325020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com /* select row and offset for dither array */ 326020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)]; 327935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 328020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com /* rescale alpha to range 0 - 256 */ 329020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int scale = SkAlpha255To256(alpha); 330935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 331020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com asm volatile ( 332020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.8 {d31}, [%[dstart]] \n\t" // load dither values 333020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d30, d31, #1 \n\t" // calc. green dither values 334020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vdup.16 d6, %[scale] \n\t" // duplicate scale into neon reg 335020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.i8 d29, #0x3f \n\t" // set up green mask 336020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmov.i8 d28, #0x1f \n\t" // set up blue mask 337020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "1: \n\t" 338020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld4.8 {d0, d1, d2, d3}, [%[src]]! \n\t" // load 8 pixels and split into argb 339020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d22, d0, #5 \n\t" // calc. red >> 5 340020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d23, d1, #6 \n\t" // calc. green >> 6 341020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u8 d24, d2, #5 \n\t" // calc. blue >> 5 342020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddl.u8 q8, d0, d31 \n\t" // add in dither to red and widen 343020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddl.u8 q9, d1, d30 \n\t" // add in dither to green and widen 344020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddl.u8 q10, d2, d31 \n\t" // add in dither to blue and widen 345020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsubw.u8 q8, q8, d22 \n\t" // sub shifted red from result 346020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsubw.u8 q9, q9, d23 \n\t" // sub shifted green from result 347020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsubw.u8 q10, q10, d24 \n\t" // sub shifted blue from result 348020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d22, q8, #3 \n\t" // shift right and narrow to 5 bits 349020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d23, q9, #2 \n\t" // shift right and narrow to 6 bits 350020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d24, q10, #3 \n\t" // shift right and narrow to 5 bits 351020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // load 8 pixels from dst, extract rgb 352020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.16 {d0, d1}, [%[dst]] \n\t" // load 8 pixels 353020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d17, q0, #5 \n\t" // shift green down to bottom 6 bits 354020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.i16 d18, q0 \n\t" // narrow to get blue as bytes 355020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshr.u16 q0, q0, #11 \n\t" // shift down to extract red 356020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vand d17, d17, d29 \n\t" // and green with green mask 357020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vand d18, d18, d28 \n\t" // and blue with blue mask 358020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmovn.i16 d16, q0 \n\t" // narrow to get red as bytes 359020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // src = {d22 (r), d23 (g), d24 (b)} 360020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // dst = {d16 (r), d17 (g), d18 (b)} 361020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // subtract dst from src and widen 362020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsubl.s8 q0, d22, d16 \n\t" // subtract red src from dst 363020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsubl.s8 q1, d23, d17 \n\t" // subtract green src from dst 364020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsubl.s8 q2, d24, d18 \n\t" // subtract blue src from dst 365020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // multiply diffs by scale and shift 366020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.i16 q0, q0, d6[0] \n\t" // multiply red by scale 367020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.i16 q1, q1, d6[0] \n\t" // multiply blue by scale 368020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmul.i16 q2, q2, d6[0] \n\t" // multiply green by scale 369020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "subs %[count], %[count], #8 \n\t" // decrement loop counter 370020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d0, q0, #8 \n\t" // shift down red by 8 and narrow 371020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d2, q1, #8 \n\t" // shift down green by 8 and narrow 372020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d4, q2, #8 \n\t" // shift down blue by 8 and narrow 373020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // add dst to result 374020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddl.s8 q0, d0, d16 \n\t" // add dst to red 375020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddl.s8 q1, d2, d17 \n\t" // add dst to green 376020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vaddl.s8 q2, d4, d18 \n\t" // add dst to blue 377020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // put result into 565 format 378020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsli.i16 q2, q1, #5 \n\t" // shift up green and insert into blue 379020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vsli.i16 q2, q0, #11 \n\t" // shift up red and insert into blue 380020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.16 {d4, d5}, [%[dst]]! \n\t" // store result 381020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "bgt 1b \n\t" // loop if count > 0 382020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [src] "+r" (src), [dst] "+r" (dst), [count] "+r" (count) 383020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [dstart] "r" (dstart), [scale] "r" (scale) 384020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : "cc", "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", "d24", "d28", "d29", "d30", "d31" 385020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 386935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 387020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_565_SCAN(y); 388935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 389020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com while((count & 7) > 0) 390020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com { 391020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColor c = *src++; 392935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 393020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int dither = DITHER_VALUE(x); 394020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int sr = SkGetPackedR32(c); 395020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int sg = SkGetPackedG32(c); 396020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int sb = SkGetPackedB32(c); 397020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sr = SkDITHER_R32To565(sr, dither); 398020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sg = SkDITHER_G32To565(sg, dither); 399020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sb = SkDITHER_B32To565(sb, dither); 400935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 401020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint16_t d = *dst; 402020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst++ = SkPackRGB16(SkAlphaBlend(sr, SkGetPackedR16(d), scale), 403020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkAlphaBlend(sg, SkGetPackedG16(d), scale), 404020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkAlphaBlend(sb, SkGetPackedB16(d), scale)); 405020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_INC_X(x); 406020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com count--; 407020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 408020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 409020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 410020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32A_Opaque_BlitRow32_neon(SkPMColor* SK_RESTRICT dst, 411020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const SkPMColor* SK_RESTRICT src, 412020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int count, U8CPU alpha) { 413020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 414020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkASSERT(255 == alpha); 415020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count > 0) { 416020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 417020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 418935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t alpha_mask; 419020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 420935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com static const uint8_t alpha_mask_setup[] = {3,3,3,3,7,7,7,7}; 421935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha_mask = vld1_u8(alpha_mask_setup); 422020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 423935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* do the NEON unrolled code */ 424935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#define UNROLL 4 425935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com while (count >= UNROLL) { 426935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t src_raw, dst_raw, dst_final; 427935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t src_raw_2, dst_raw_2, dst_final_2; 428020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 42958cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org /* The two prefetches below may make the code slighlty 43058cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org * slower for small values of count but are worth having 43158cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org * in the general case. 43258cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org */ 43358cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org __builtin_prefetch(src+32); 43458cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org __builtin_prefetch(dst+32); 43558cf9feb120cb0ba7009b617f43470d519aef5a9commit-bot@chromium.org 436935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* get the source */ 437935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src_raw = vreinterpret_u8_u32(vld1_u32(src)); 438935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if UNROLL > 2 439935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src_raw_2 = vreinterpret_u8_u32(vld1_u32(src+2)); 440020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 441020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 442935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* get and hold the dst too */ 443935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_raw = vreinterpret_u8_u32(vld1_u32(dst)); 444935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if UNROLL > 2 445935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_raw_2 = vreinterpret_u8_u32(vld1_u32(dst+2)); 446020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 447020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 448935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* 1st and 2nd bits of the unrolling */ 449935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 450935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t dst_cooked; 451935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t dst_wide; 452935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t alpha_narrow; 453935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t alpha_wide; 454020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 455935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* get the alphas spread out properly */ 456935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha_narrow = vtbl1_u8(src_raw, alpha_mask); 457935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha_wide = vsubw_u8(vdupq_n_u16(256), alpha_narrow); 458020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 459935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* spread the dest */ 460935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_wide = vmovl_u8(dst_raw); 461020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 462935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* alpha mul the dest */ 463935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_wide = vmulq_u16 (dst_wide, alpha_wide); 464935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_cooked = vshrn_n_u16(dst_wide, 8); 465020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 466935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sum -- ignoring any byte lane overflows */ 467935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_final = vadd_u8(src_raw, dst_cooked); 468935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 469020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 470935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if UNROLL > 2 471935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* the 3rd and 4th bits of our unrolling */ 472935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 473935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t dst_cooked; 474935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t dst_wide; 475935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t alpha_narrow; 476935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t alpha_wide; 477020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 478935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha_narrow = vtbl1_u8(src_raw_2, alpha_mask); 479935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha_wide = vsubw_u8(vdupq_n_u16(256), alpha_narrow); 480020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 481935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* spread the dest */ 482935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_wide = vmovl_u8(dst_raw_2); 483020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 484935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* alpha mul the dest */ 485935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_wide = vmulq_u16 (dst_wide, alpha_wide); 486935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_cooked = vshrn_n_u16(dst_wide, 8); 487020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 488935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sum -- ignoring any byte lane overflows */ 489935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_final_2 = vadd_u8(src_raw_2, dst_cooked); 490935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 491020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 492020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 493935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com vst1_u32(dst, vreinterpret_u32_u8(dst_final)); 494935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if UNROLL > 2 495935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com vst1_u32(dst+2, vreinterpret_u32_u8(dst_final_2)); 496020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 497020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 498935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src += UNROLL; 499935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst += UNROLL; 500935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com count -= UNROLL; 501935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 502935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#undef UNROLL 503020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 504935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* do any residual iterations */ 505020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com while (--count >= 0) { 506020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst = SkPMSrcOver(*src, *dst); 507020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com src += 1; 508020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst += 1; 509020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 510020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 511020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 512020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 5139fba280106226e22454cb36bc90aa37badd37976djsollen@google.comvoid S32A_Opaque_BlitRow32_neon_src_alpha(SkPMColor* SK_RESTRICT dst, 5149fba280106226e22454cb36bc90aa37badd37976djsollen@google.com const SkPMColor* SK_RESTRICT src, 5159fba280106226e22454cb36bc90aa37badd37976djsollen@google.com int count, U8CPU alpha) { 5169fba280106226e22454cb36bc90aa37badd37976djsollen@google.com SkASSERT(255 == alpha); 5179fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5189fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if (count <= 0) 5199fba280106226e22454cb36bc90aa37badd37976djsollen@google.com return; 5209fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5219fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* Use these to check if src is transparent or opaque */ 5229fba280106226e22454cb36bc90aa37badd37976djsollen@google.com const unsigned int ALPHA_OPAQ = 0xFF000000; 5239fba280106226e22454cb36bc90aa37badd37976djsollen@google.com const unsigned int ALPHA_TRANS = 0x00FFFFFF; 5249fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5259fba280106226e22454cb36bc90aa37badd37976djsollen@google.com#define UNROLL 4 5269fba280106226e22454cb36bc90aa37badd37976djsollen@google.com const SkPMColor* SK_RESTRICT src_end = src + count - (UNROLL + 1); 5279fba280106226e22454cb36bc90aa37badd37976djsollen@google.com const SkPMColor* SK_RESTRICT src_temp = src; 5289fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5299fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* set up the NEON variables */ 5309fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint8x8_t alpha_mask; 5319fba280106226e22454cb36bc90aa37badd37976djsollen@google.com static const uint8_t alpha_mask_setup[] = {3,3,3,3,7,7,7,7}; 5329fba280106226e22454cb36bc90aa37badd37976djsollen@google.com alpha_mask = vld1_u8(alpha_mask_setup); 5339fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5349fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint8x8_t src_raw, dst_raw, dst_final; 5359fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint8x8_t src_raw_2, dst_raw_2, dst_final_2; 5369fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint8x8_t dst_cooked; 5379fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint16x8_t dst_wide; 5389fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint8x8_t alpha_narrow; 5399fba280106226e22454cb36bc90aa37badd37976djsollen@google.com uint16x8_t alpha_wide; 5409fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5419fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* choose the first processing type */ 5429fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if( src >= src_end) 5439fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto TAIL; 5449fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src <= ALPHA_TRANS) 5459fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_0; 5469fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src >= ALPHA_OPAQ) 5479fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_255; 5489fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* fall-thru */ 5499fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5509fba280106226e22454cb36bc90aa37badd37976djsollen@google.comALPHA_1_TO_254: 5519fba280106226e22454cb36bc90aa37badd37976djsollen@google.com do { 5529fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5539fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* get the source */ 5549fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src_raw = vreinterpret_u8_u32(vld1_u32(src)); 5559fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src_raw_2 = vreinterpret_u8_u32(vld1_u32(src+2)); 5569fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5579fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* get and hold the dst too */ 5589fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_raw = vreinterpret_u8_u32(vld1_u32(dst)); 5599fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_raw_2 = vreinterpret_u8_u32(vld1_u32(dst+2)); 5609fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5619fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5629fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* get the alphas spread out properly */ 5639fba280106226e22454cb36bc90aa37badd37976djsollen@google.com alpha_narrow = vtbl1_u8(src_raw, alpha_mask); 5649fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* reflect SkAlpha255To256() semantics a+1 vs a+a>>7 */ 5659fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* we collapsed (255-a)+1 ... */ 5669fba280106226e22454cb36bc90aa37badd37976djsollen@google.com alpha_wide = vsubw_u8(vdupq_n_u16(256), alpha_narrow); 5679fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5689fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* spread the dest */ 5699fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_wide = vmovl_u8(dst_raw); 5709fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5719fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* alpha mul the dest */ 5729fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_wide = vmulq_u16 (dst_wide, alpha_wide); 5739fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_cooked = vshrn_n_u16(dst_wide, 8); 5749fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5759fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* sum -- ignoring any byte lane overflows */ 5769fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_final = vadd_u8(src_raw, dst_cooked); 5779fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5789fba280106226e22454cb36bc90aa37badd37976djsollen@google.com alpha_narrow = vtbl1_u8(src_raw_2, alpha_mask); 5799fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* reflect SkAlpha255To256() semantics a+1 vs a+a>>7 */ 5809fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* we collapsed (255-a)+1 ... */ 5819fba280106226e22454cb36bc90aa37badd37976djsollen@google.com alpha_wide = vsubw_u8(vdupq_n_u16(256), alpha_narrow); 5829fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5839fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* spread the dest */ 5849fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_wide = vmovl_u8(dst_raw_2); 5859fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5869fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* alpha mul the dest */ 5879fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_wide = vmulq_u16 (dst_wide, alpha_wide); 5889fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_cooked = vshrn_n_u16(dst_wide, 8); 5899fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5909fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* sum -- ignoring any byte lane overflows */ 5919fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst_final_2 = vadd_u8(src_raw_2, dst_cooked); 5929fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5939fba280106226e22454cb36bc90aa37badd37976djsollen@google.com vst1_u32(dst, vreinterpret_u32_u8(dst_final)); 5949fba280106226e22454cb36bc90aa37badd37976djsollen@google.com vst1_u32(dst+2, vreinterpret_u32_u8(dst_final_2)); 5959fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5969fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src += UNROLL; 5979fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst += UNROLL; 5989fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 5999fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* if 2 of the next pixels aren't between 1 and 254 6009fba280106226e22454cb36bc90aa37badd37976djsollen@google.com it might make sense to go to the optimized loops */ 6019fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if((src[0] <= ALPHA_TRANS && src[1] <= ALPHA_TRANS) || (src[0] >= ALPHA_OPAQ && src[1] >= ALPHA_OPAQ)) 6029fba280106226e22454cb36bc90aa37badd37976djsollen@google.com break; 6039fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6049fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } while(src < src_end); 6059fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6069fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if (src >= src_end) 6079fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto TAIL; 6089fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6099fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(src[0] >= ALPHA_OPAQ && src[1] >= ALPHA_OPAQ) 6109fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_255; 6119fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6129fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /*fall-thru*/ 6139fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6149fba280106226e22454cb36bc90aa37badd37976djsollen@google.comALPHA_0: 6159fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6169fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /*In this state, we know the current alpha is 0 and 6179fba280106226e22454cb36bc90aa37badd37976djsollen@google.com we optimize for the next alpha also being zero. */ 6189fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src_temp = src; //so we don't have to increment dst every time 6199fba280106226e22454cb36bc90aa37badd37976djsollen@google.com do { 6209fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*(++src) > ALPHA_TRANS) 6219fba280106226e22454cb36bc90aa37badd37976djsollen@google.com break; 6229fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*(++src) > ALPHA_TRANS) 6239fba280106226e22454cb36bc90aa37badd37976djsollen@google.com break; 6249fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*(++src) > ALPHA_TRANS) 6259fba280106226e22454cb36bc90aa37badd37976djsollen@google.com break; 6269fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*(++src) > ALPHA_TRANS) 6279fba280106226e22454cb36bc90aa37badd37976djsollen@google.com break; 6289fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } while(src < src_end); 6299fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6309fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst += (src - src_temp); 6319fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6329fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* no longer alpha 0, so determine where to go next. */ 6339fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if( src >= src_end) 6349fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto TAIL; 6359fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src >= ALPHA_OPAQ) 6369fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_255; 6379fba280106226e22454cb36bc90aa37badd37976djsollen@google.com else 6389fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_1_TO_254; 6399fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6409fba280106226e22454cb36bc90aa37badd37976djsollen@google.comALPHA_255: 6419fba280106226e22454cb36bc90aa37badd37976djsollen@google.com while((src[0] & src[1] & src[2] & src[3]) >= ALPHA_OPAQ) { 6429fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst[0]=src[0]; 6439fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst[1]=src[1]; 6449fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst[2]=src[2]; 6459fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst[3]=src[3]; 6469fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src+=UNROLL; 6479fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst+=UNROLL; 6489fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(src >= src_end) 6499fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto TAIL; 6509fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6519fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6529fba280106226e22454cb36bc90aa37badd37976djsollen@google.com //Handle remainder. 6539fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src >= ALPHA_OPAQ) { *dst++ = *src++; 6549fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src >= ALPHA_OPAQ) { *dst++ = *src++; 6559fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src >= ALPHA_OPAQ) { *dst++ = *src++; } 6569fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6579fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6589fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6599fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if( src >= src_end) 6609fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto TAIL; 6619fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if(*src <= ALPHA_TRANS) 6629fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_0; 6639fba280106226e22454cb36bc90aa37badd37976djsollen@google.com else 6649fba280106226e22454cb36bc90aa37badd37976djsollen@google.com goto ALPHA_1_TO_254; 6659fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6669fba280106226e22454cb36bc90aa37badd37976djsollen@google.comTAIL: 6679fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* do any residual iterations */ 6689fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src_end += UNROLL + 1; //goto the real end 6699fba280106226e22454cb36bc90aa37badd37976djsollen@google.com while(src != src_end) { 6709fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if( *src != 0 ) { 6719fba280106226e22454cb36bc90aa37badd37976djsollen@google.com if( *src >= ALPHA_OPAQ ) { 6729fba280106226e22454cb36bc90aa37badd37976djsollen@google.com *dst = *src; 6739fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6749fba280106226e22454cb36bc90aa37badd37976djsollen@google.com else { 6759fba280106226e22454cb36bc90aa37badd37976djsollen@google.com *dst = SkPMSrcOver(*src, *dst); 6769fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6779fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6789fba280106226e22454cb36bc90aa37badd37976djsollen@google.com src++; 6799fba280106226e22454cb36bc90aa37badd37976djsollen@google.com dst++; 6809fba280106226e22454cb36bc90aa37badd37976djsollen@google.com } 6819fba280106226e22454cb36bc90aa37badd37976djsollen@google.com 6829fba280106226e22454cb36bc90aa37badd37976djsollen@google.com#undef UNROLL 6839fba280106226e22454cb36bc90aa37badd37976djsollen@google.com return; 6849fba280106226e22454cb36bc90aa37badd37976djsollen@google.com} 685020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 686020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/* Neon version of S32_Blend_BlitRow32() 687020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * portable version is in src/core/SkBlitRow_D32.cpp 688020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com */ 689020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32_Blend_BlitRow32_neon(SkPMColor* SK_RESTRICT dst, 690020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const SkPMColor* SK_RESTRICT src, 691020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int count, U8CPU alpha) { 692020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkASSERT(alpha <= 255); 693020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count > 0) { 694020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint16_t src_scale = SkAlpha255To256(alpha); 695020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint16_t dst_scale = 256 - src_scale; 696020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 697935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* run them N at a time through the NEON unit */ 698935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* note that each 1 is 4 bytes, each treated exactly the same, 699935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * so we can work under that guise. We *do* know that the src&dst 700935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * will be 32-bit aligned quantities, so we can specify that on 701935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * the load/store ops and do a neon 'reinterpret' to get us to 702935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * byte-sized (pun intended) pieces that we widen/multiply/shift 703935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * we're limited at 128 bits in the wide ops, which is 8x16bits 704935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * or a pair of 32 bit src/dsts. 705935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com */ 706935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* we *could* manually unroll this loop so that we load 128 bits 707935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * (as a pair of 64s) from each of src and dst, processing them 708935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * in pieces. This might give us a little better management of 709935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * the memory latency, but my initial attempts here did not 710935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * produce an instruction stream that looked all that nice. 711935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com */ 712935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#define UNROLL 2 713935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com while (count >= UNROLL) { 714935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t src_raw, dst_raw, dst_final; 715935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t src_wide, dst_wide; 716935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 717935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* get 64 bits of src, widen it, multiply by src_scale */ 718935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src_raw = vreinterpret_u8_u32(vld1_u32(src)); 719935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src_wide = vmovl_u8(src_raw); 720935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* gcc hoists vdupq_n_u16(), better than using vmulq_n_u16() */ 721935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src_wide = vmulq_u16 (src_wide, vdupq_n_u16(src_scale)); 722935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 723935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* ditto with dst */ 724935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_raw = vreinterpret_u8_u32(vld1_u32(dst)); 725935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_wide = vmovl_u8(dst_raw); 726935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 727935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* combine add with dst multiply into mul-accumulate */ 728935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_wide = vmlaq_u16(src_wide, dst_wide, vdupq_n_u16(dst_scale)); 729935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 730935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_final = vshrn_n_u16(dst_wide, 8); 731935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com vst1_u32(dst, vreinterpret_u32_u8(dst_final)); 732935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 733935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src += UNROLL; 734935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst += UNROLL; 735935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com count -= UNROLL; 736935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 737935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* RBE: well, i don't like how gcc manages src/dst across the above 738935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * loop it's constantly calculating src+bias, dst+bias and it only 739935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * adjusts the real ones when we leave the loop. Not sure why 740935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * it's "hoisting down" (hoisting implies above in my lexicon ;)) 741935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * the adjustments to src/dst/count, but it does... 742935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * (might be SSA-style internal logic... 743935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com */ 744935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 745935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if UNROLL == 2 746935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com if (count == 1) { 747020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst = SkAlphaMulQ(*src, src_scale) + SkAlphaMulQ(*dst, dst_scale); 748935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 749020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#else 750935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com if (count > 0) { 751020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com do { 752020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst = SkAlphaMulQ(*src, src_scale) + SkAlphaMulQ(*dst, dst_scale); 753020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com src += 1; 754020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst += 1; 755020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } while (--count > 0); 756935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 757020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 758020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 759935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#undef UNROLL 760020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 761020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 762020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 763c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.orgvoid S32A_Blend_BlitRow32_neon(SkPMColor* SK_RESTRICT dst, 764c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org const SkPMColor* SK_RESTRICT src, 765c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org int count, U8CPU alpha) { 766c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 767c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org SkASSERT(255 >= alpha); 768c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 769c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org if (count <= 0) { 770c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org return; 771c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org } 772c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 773c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org unsigned alpha256 = SkAlpha255To256(alpha); 774c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 775c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // First deal with odd counts 776c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org if (count & 1) { 777c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org uint8x8_t vsrc = vdup_n_u8(0), vdst = vdup_n_u8(0), vres; 778c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org uint16x8_t vdst_wide, vsrc_wide; 779c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org unsigned dst_scale; 780c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 781c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Load 782c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc = vreinterpret_u8_u32(vld1_lane_u32(src, vreinterpret_u32_u8(vsrc), 0)); 783c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst = vreinterpret_u8_u32(vld1_lane_u32(dst, vreinterpret_u32_u8(vdst), 0)); 784c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 785c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Calc dst_scale 786c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org dst_scale = vget_lane_u8(vsrc, 3); 787c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org dst_scale *= alpha256; 788c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org dst_scale >>= 8; 789c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org dst_scale = 256 - dst_scale; 790c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 791c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Process src 792c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc_wide = vmovl_u8(vsrc); 793c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc_wide = vmulq_n_u16(vsrc_wide, alpha256); 794c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 795c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Process dst 796c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_wide = vmovl_u8(vdst); 797c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_wide = vmulq_n_u16(vdst_wide, dst_scale); 798c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 799c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Combine 800c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vres = vshrn_n_u16(vdst_wide, 8) + vshrn_n_u16(vsrc_wide, 8); 801c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 802c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vst1_lane_u32(dst, vreinterpret_u32_u8(vres), 0); 803c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org dst++; 804c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org src++; 805c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org count--; 806c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org } 807c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 808c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org if (count) { 809c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org uint8x8_t alpha_mask; 810c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org static const uint8_t alpha_mask_setup[] = {3,3,3,3,7,7,7,7}; 811c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org alpha_mask = vld1_u8(alpha_mask_setup); 812c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 813c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org do { 814c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 815c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org uint8x8_t vsrc, vdst, vres, vsrc_alphas; 816c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org uint16x8_t vdst_wide, vsrc_wide, vsrc_scale, vdst_scale; 817c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 818c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org __builtin_prefetch(src+32); 819c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org __builtin_prefetch(dst+32); 820c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 821c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Load 822c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc = vreinterpret_u8_u32(vld1_u32(src)); 823c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst = vreinterpret_u8_u32(vld1_u32(dst)); 824c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 825c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Prepare src_scale 826c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc_scale = vdupq_n_u16(alpha256); 827c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 828c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Calc dst_scale 829c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc_alphas = vtbl1_u8(vsrc, alpha_mask); 830c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_scale = vmovl_u8(vsrc_alphas); 831c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_scale *= vsrc_scale; 832c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_scale = vshrq_n_u16(vdst_scale, 8); 833c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_scale = vsubq_u16(vdupq_n_u16(256), vdst_scale); 834c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 835c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Process src 836c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc_wide = vmovl_u8(vsrc); 837c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vsrc_wide *= vsrc_scale; 838c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 839c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Process dst 840c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_wide = vmovl_u8(vdst); 841c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vdst_wide *= vdst_scale; 842c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 843c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org // Combine 844c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vres = vshrn_n_u16(vdst_wide, 8) + vshrn_n_u16(vsrc_wide, 8); 845c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 846c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org vst1_u32(dst, vreinterpret_u32_u8(vres)); 847c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 848c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org src += 2; 849c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org dst += 2; 850c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org count -= 2; 851c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org } while(count); 852c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org } 853c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org} 854c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org 855020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/////////////////////////////////////////////////////////////////////////////// 856020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 857935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#undef DEBUG_OPAQUE_DITHER 858020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 859935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if defined(DEBUG_OPAQUE_DITHER) 860020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comstatic void showme8(char *str, void *p, int len) 861020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com{ 862935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com static char buf[256]; 863935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com char tbuf[32]; 864935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int i; 865935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com char *pc = (char*) p; 866935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sprintf(buf,"%8s:", str); 867935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com for(i=0;i<len;i++) { 868935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sprintf(tbuf, " %02x", pc[i]); 869935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com strcat(buf, tbuf); 870935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 871935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkDebugf("%s\n", buf); 872020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 873020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comstatic void showme16(char *str, void *p, int len) 874020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com{ 875935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com static char buf[256]; 876935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com char tbuf[32]; 877935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int i; 878935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16_t *pc = (uint16_t*) p; 879935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sprintf(buf,"%8s:", str); 880935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com len = (len / sizeof(uint16_t)); /* passed as bytes */ 881935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com for(i=0;i<len;i++) { 882935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sprintf(tbuf, " %04x", pc[i]); 883935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com strcat(buf, tbuf); 884935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 885935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkDebugf("%s\n", buf); 886020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 887020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 888020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 889020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32A_D565_Opaque_Dither_neon (uint16_t * SK_RESTRICT dst, 890020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const SkPMColor* SK_RESTRICT src, 891020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int count, U8CPU alpha, int x, int y) { 892020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkASSERT(255 == alpha); 893020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 894935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#define UNROLL 8 895020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 896020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count >= UNROLL) { 897935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t dbase; 898935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 899935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if defined(DEBUG_OPAQUE_DITHER) 900935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16_t tmpbuf[UNROLL]; 901935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int td[UNROLL]; 902935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int tdv[UNROLL]; 903935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int ta[UNROLL]; 904935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int tap[UNROLL]; 905935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16_t in_dst[UNROLL]; 906935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int offset = 0; 907935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int noisy = 0; 908020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 909020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 910935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)]; 911935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dbase = vld1_u8(dstart); 912020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 913020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com do { 914935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t sr, sg, sb, sa, d; 915935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t dst8, scale8, alpha8; 916935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t dst_r, dst_g, dst_b; 917935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 918935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if defined(DEBUG_OPAQUE_DITHER) 919935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* calculate 8 elements worth into a temp buffer */ 920935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 921935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int my_y = y; 922935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int my_x = x; 923935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkPMColor* my_src = (SkPMColor*)src; 924935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16_t* my_dst = dst; 925935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int i; 926020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 927020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_565_SCAN(my_y); 928020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com for(i=0;i<UNROLL;i++) { 929020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColor c = *my_src++; 930020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColorAssert(c); 931020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (c) { 932020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned a = SkGetPackedA32(c); 933935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 934020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int d = SkAlphaMul(DITHER_VALUE(my_x), SkAlpha255To256(a)); 935935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com tdv[i] = DITHER_VALUE(my_x); 936935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com ta[i] = a; 937935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com tap[i] = SkAlpha255To256(a); 938935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com td[i] = d; 939935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 940020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned sr = SkGetPackedR32(c); 941020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned sg = SkGetPackedG32(c); 942020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned sb = SkGetPackedB32(c); 943020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sr = SkDITHER_R32_FOR_565(sr, d); 944020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sg = SkDITHER_G32_FOR_565(sg, d); 945020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sb = SkDITHER_B32_FOR_565(sb, d); 946935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 947020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint32_t src_expanded = (sg << 24) | (sr << 13) | (sb << 2); 948020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint32_t dst_expanded = SkExpand_rgb_16(*my_dst); 949020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst_expanded = dst_expanded * (SkAlpha255To256(255 - a) >> 3); 950020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // now src and dst expanded are in g:11 r:10 x:1 b:10 951020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com tmpbuf[i] = SkCompact_rgb_16((src_expanded + dst_expanded) >> 5); 952935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com td[i] = d; 953020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 954020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } else { 955935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com tmpbuf[i] = *my_dst; 956935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com ta[i] = tdv[i] = td[i] = 0xbeef; 957935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 958935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com in_dst[i] = *my_dst; 959020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com my_dst += 1; 960020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_INC_X(my_x); 961020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 962935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 963020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 964020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 965935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* source is in ABGR */ 966935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 967935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d0 asm("d0"); 968935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d1 asm("d1"); 969935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d2 asm("d2"); 970935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d3 asm("d3"); 971020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 972935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com asm ("vld4.8 {d0-d3},[%4] /* r=%P0 g=%P1 b=%P2 a=%P3 */" 973935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3) 974935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com : "r" (src) 975020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 976935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sr = d0; sg = d1; sb = d2; sa = d3; 977935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 978020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 979935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* calculate 'd', which will be 0..7 */ 980935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* dbase[] is 0..7; alpha is 0..256; 16 bits suffice */ 981020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#if defined(SK_BUILD_FOR_ANDROID) 982935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* SkAlpha255To256() semantic a+1 vs a+a>>7 */ 983935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha8 = vaddw_u8(vmovl_u8(sa), vdup_n_u8(1)); 984020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#else 985935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha8 = vaddw_u8(vmovl_u8(sa), vshr_n_u8(sa, 7)); 986020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 987935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com alpha8 = vmulq_u16(alpha8, vmovl_u8(dbase)); 988935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com d = vshrn_n_u16(alpha8, 8); /* narrowing too */ 989935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 990935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sr = sr - (sr>>5) + d */ 991935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* watching for 8-bit overflow. d is 0..7; risky range of 992935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * sr is >248; and then (sr>>5) is 7 so it offsets 'd'; 993935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * safe as long as we do ((sr-sr>>5) + d) */ 994935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sr = vsub_u8(sr, vshr_n_u8(sr, 5)); 995935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sr = vadd_u8(sr, d); 996935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 997935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sb = sb - (sb>>5) + d */ 998935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sb = vsub_u8(sb, vshr_n_u8(sb, 5)); 999935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sb = vadd_u8(sb, d); 1000935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1001935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sg = sg - (sg>>6) + d>>1; similar logic for overflows */ 1002935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sg = vsub_u8(sg, vshr_n_u8(sg, 6)); 1003935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sg = vadd_u8(sg, vshr_n_u8(d,1)); 1004935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1005935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* need to pick up 8 dst's -- at 16 bits each, 128 bits */ 1006935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vld1q_u16(dst); 1007935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_b = vandq_u16(dst8, vdupq_n_u16(0x001F)); 1008935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_g = vandq_u16(vshrq_n_u16(dst8,5), vdupq_n_u16(0x003F)); 1009935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_r = vshrq_n_u16(dst8,11); /* clearing hi bits */ 1010935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1011935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* blend */ 1012020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#if 1 1013935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* SkAlpha255To256() semantic a+1 vs a+a>>7 */ 1014935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* originally 255-sa + 1 */ 1015935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com scale8 = vsubw_u8(vdupq_n_u16(256), sa); 1016020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#else 1017935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com scale8 = vsubw_u8(vdupq_n_u16(255), sa); 1018935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com scale8 = vaddq_u16(scale8, vshrq_n_u16(scale8, 7)); 1019020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 1020020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1021020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#if 1 1022935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* combine the addq and mul, save 3 insns */ 1023935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com scale8 = vshrq_n_u16(scale8, 3); 1024935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_b = vmlaq_u16(vshll_n_u8(sb,2), dst_b, scale8); 1025935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_g = vmlaq_u16(vshll_n_u8(sg,3), dst_g, scale8); 1026935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_r = vmlaq_u16(vshll_n_u8(sr,2), dst_r, scale8); 1027020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#else 1028935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* known correct, but +3 insns over above */ 1029935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com scale8 = vshrq_n_u16(scale8, 3); 1030935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_b = vmulq_u16(dst_b, scale8); 1031935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_g = vmulq_u16(dst_g, scale8); 1032935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_r = vmulq_u16(dst_r, scale8); 1033935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1034935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* combine */ 1035935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* NB: vshll widens, need to preserve those bits */ 1036935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_b = vaddq_u16(dst_b, vshll_n_u8(sb,2)); 1037935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_g = vaddq_u16(dst_g, vshll_n_u8(sg,3)); 1038935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst_r = vaddq_u16(dst_r, vshll_n_u8(sr,2)); 1039020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 1040020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1041935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* repack to store */ 1042935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vandq_u16(vshrq_n_u16(dst_b, 5), vdupq_n_u16(0x001F)); 1043935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dst_g, 5), 5); 1044935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dst_r,5), 11); 1045935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1046935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com vst1q_u16(dst, dst8); 1047935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1048935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if defined(DEBUG_OPAQUE_DITHER) 1049935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* verify my 8 elements match the temp buffer */ 1050935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 1051935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int i, bad=0; 1052935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com static int invocation; 1053935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1054935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com for (i=0;i<UNROLL;i++) 1055935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com if (tmpbuf[i] != dst[i]) bad=1; 1056935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com if (bad) { 1057935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkDebugf("BAD S32A_D565_Opaque_Dither_neon(); invocation %d offset %d\n", 1058935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com invocation, offset); 1059935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkDebugf(" alpha 0x%x\n", alpha); 1060935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com for (i=0;i<UNROLL;i++) 1061935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkDebugf("%2d: %s %04x w %04x id %04x s %08x d %04x %04x %04x %04x\n", 1062935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com i, ((tmpbuf[i] != dst[i])?"BAD":"got"), 1063935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst[i], tmpbuf[i], in_dst[i], src[i], td[i], tdv[i], tap[i], ta[i]); 1064935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1065935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme16("alpha8", &alpha8, sizeof(alpha8)); 1066935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme16("scale8", &scale8, sizeof(scale8)); 1067935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme8("d", &d, sizeof(d)); 1068935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme16("dst8", &dst8, sizeof(dst8)); 1069935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme16("dst_b", &dst_b, sizeof(dst_b)); 1070935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme16("dst_g", &dst_g, sizeof(dst_g)); 1071935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme16("dst_r", &dst_r, sizeof(dst_r)); 1072935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme8("sb", &sb, sizeof(sb)); 1073935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme8("sg", &sg, sizeof(sg)); 1074935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com showme8("sr", &sr, sizeof(sr)); 1075935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1076935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* cop out */ 1077935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com return; 1078935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1079935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com offset += UNROLL; 1080935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com invocation++; 1081935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1082020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 1083020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1084020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst += UNROLL; 1085935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src += UNROLL; 1086935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com count -= UNROLL; 1087935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* skip x += UNROLL, since it's unchanged mod-4 */ 1088020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } while (count >= UNROLL); 1089020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1090935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#undef UNROLL 1091020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1092020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com /* residuals */ 1093020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count > 0) { 1094020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_565_SCAN(y); 1095020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com do { 1096020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColor c = *src++; 1097020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColorAssert(c); 1098020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (c) { 1099020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned a = SkGetPackedA32(c); 1100935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1101020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // dither and alpha are just temporary variables to work-around 1102020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // an ICE in debug. 1103020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned dither = DITHER_VALUE(x); 1104020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned alpha = SkAlpha255To256(a); 1105020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int d = SkAlphaMul(dither, alpha); 1106935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1107020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned sr = SkGetPackedR32(c); 1108020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned sg = SkGetPackedG32(c); 1109020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned sb = SkGetPackedB32(c); 1110020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sr = SkDITHER_R32_FOR_565(sr, d); 1111020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sg = SkDITHER_G32_FOR_565(sg, d); 1112020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sb = SkDITHER_B32_FOR_565(sb, d); 1113935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1114020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint32_t src_expanded = (sg << 24) | (sr << 13) | (sb << 2); 1115020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com uint32_t dst_expanded = SkExpand_rgb_16(*dst); 1116020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst_expanded = dst_expanded * (SkAlpha255To256(255 - a) >> 3); 1117020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // now src and dst expanded are in g:11 r:10 x:1 b:10 1118020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst = SkCompact_rgb_16((src_expanded + dst_expanded) >> 5); 1119020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1120020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst += 1; 1121020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_INC_X(x); 1122020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } while (--count != 0); 1123020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1124020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 1125020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1126020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/////////////////////////////////////////////////////////////////////////////// 1127020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1128020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/* 2009/10/27: RBE says "a work in progress"; debugging says ok; 1129020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * speedup untested, but ARM version is 26 insns/iteration and 1130020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * this NEON version is 21 insns/iteration-of-8 (2.62insns/element) 1131020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * which is 10x the native version; that's pure instruction counts, 1132020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com * not accounting for any instruction or memory latencies. 1133020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com */ 1134020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1135935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#undef DEBUG_S32_OPAQUE_DITHER 1136020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1137020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid S32_D565_Opaque_Dither_neon(uint16_t* SK_RESTRICT dst, 1138020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com const SkPMColor* SK_RESTRICT src, 1139020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com int count, U8CPU alpha, int x, int y) { 1140020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkASSERT(255 == alpha); 1141020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1142935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#define UNROLL 8 1143020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count >= UNROLL) { 1144935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint8x8_t d; 1145935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)]; 1146935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com d = vld1_u8(dstart); 1147935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1148935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com while (count >= UNROLL) { 1149d8f6ff71cd1c36a388eec93282a1fd1c0877a7f5djsollen@google.com uint8x8_t sr, sg, sb; 1150d8f6ff71cd1c36a388eec93282a1fd1c0877a7f5djsollen@google.com uint16x8_t dr, dg, db; 1151935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16x8_t dst8; 1152935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1153935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* source is in ABGR ordering (R == lsb) */ 1154935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 1155935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d0 asm("d0"); 1156935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d1 asm("d1"); 1157935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d2 asm("d2"); 1158935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com register uint8x8_t d3 asm("d3"); 1159935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1160935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com asm ("vld4.8 {d0-d3},[%4] /* r=%P0 g=%P1 b=%P2 a=%P3 */" 1161935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3) 1162935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com : "r" (src) 1163020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 1164d8f6ff71cd1c36a388eec93282a1fd1c0877a7f5djsollen@google.com sr = d0; sg = d1; sb = d2; 1165935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1166935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* XXX: if we want to prefetch, hide it in the above asm() 1167935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * using the gcc __builtin_prefetch(), the prefetch will 1168935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * fall to the bottom of the loop -- it won't stick up 1169935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com * at the top of the loop, just after the vld4. 1170935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com */ 1171935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1172935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sr = sr - (sr>>5) + d */ 1173935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sr = vsub_u8(sr, vshr_n_u8(sr, 5)); 1174935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dr = vaddl_u8(sr, d); 1175935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1176935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sb = sb - (sb>>5) + d */ 1177935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sb = vsub_u8(sb, vshr_n_u8(sb, 5)); 1178935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com db = vaddl_u8(sb, d); 1179935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1180935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* sg = sg - (sg>>6) + d>>1; similar logic for overflows */ 1181935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com sg = vsub_u8(sg, vshr_n_u8(sg, 6)); 1182935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dg = vaddl_u8(sg, vshr_n_u8(d,1)); 1183935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* XXX: check that the "d>>1" here is hoisted */ 1184935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1185935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* pack high bits of each into 565 format (rgb, b is lsb) */ 1186935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vshrq_n_u16(db, 3); 1187935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dg, 2), 5); 1188935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dr,3), 11); 1189935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1190935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* store it */ 1191935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com vst1q_u16(dst, dst8); 1192935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com 1193935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#if defined(DEBUG_S32_OPAQUE_DITHER) 1194935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com /* always good to know if we generated good results */ 1195935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com { 1196935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com int i, myx = x, myy = y; 1197935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com DITHER_565_SCAN(myy); 1198935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com for (i=0;i<UNROLL;i++) { 1199935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkPMColor c = src[i]; 1200935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com unsigned dither = DITHER_VALUE(myx); 1201935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com uint16_t val = SkDitherRGB32To565(c, dither); 1202935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com if (val != dst[i]) { 1203935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com SkDebugf("RBE: src %08x dither %02x, want %04x got %04x dbas[i] %02x\n", 1204935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com c, dither, val, dst[i], dstart[i]); 1205935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1206935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com DITHER_INC_X(myx); 1207935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1208935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1209020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com#endif 1210020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1211935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com dst += UNROLL; 1212935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com src += UNROLL; 1213935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com count -= UNROLL; 1214935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com x += UNROLL; /* probably superfluous */ 1215935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com } 1216020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1217935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com#undef UNROLL 1218020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1219020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com /* residuals */ 1220020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count > 0) { 1221020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_565_SCAN(y); 1222020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com do { 1223020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColor c = *src++; 1224020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColorAssert(c); 1225020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkASSERT(SkGetPackedA32(c) == 255); 1226020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1227020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned dither = DITHER_VALUE(x); 1228020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst++ = SkDitherRGB32To565(c, dither); 1229020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com DITHER_INC_X(x); 1230020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } while (--count != 0); 1231020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1232020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 1233020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1234020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comvoid Color32_arm_neon(SkPMColor* dst, const SkPMColor* src, int count, 1235020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com SkPMColor color) { 1236020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count <= 0) { 1237020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com return; 1238020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1239020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1240020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (0 == color) { 1241020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (src != dst) { 1242020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com memcpy(dst, src, count * sizeof(SkPMColor)); 1243020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1244020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com return; 1245020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1246020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1247020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned colorA = SkGetPackedA32(color); 1248020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (255 == colorA) { 1249020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com sk_memset32(dst, color, count); 1250020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } else { 1251020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com unsigned scale = 256 - SkAlpha255To256(colorA); 1252020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1253020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com if (count >= 8) { 1254020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // at the end of this assembly, count will have been decremented 1255020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // to a negative value. That is, if count mod 8 = x, it will be 1256020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // -8 +x coming out. 1257020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com asm volatile ( 1258020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com PLD128(src, 0) 1259020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1260020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vdup.32 q0, %[color] \n\t" 1261020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1262020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com PLD128(src, 128) 1263020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1264020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // scale numerical interval [0-255], so load as 8 bits 1265020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vdup.8 d2, %[scale] \n\t" 1266020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1267020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com PLD128(src, 256) 1268020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1269020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "subs %[count], %[count], #8 \n\t" 1270020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1271020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com PLD128(src, 384) 1272020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1273020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "Loop_Color32: \n\t" 1274020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1275020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // load src color, 8 pixels, 4 64 bit registers 1276020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // (and increment src). 1277020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vld1.32 {d4-d7}, [%[src]]! \n\t" 1278020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1279020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com PLD128(src, 384) 1280020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1281020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // multiply long by scale, 64 bits at a time, 1282020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // destination into a 128 bit register. 1283020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmull.u8 q4, d4, d2 \n\t" 1284020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmull.u8 q5, d5, d2 \n\t" 1285020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmull.u8 q6, d6, d2 \n\t" 1286020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vmull.u8 q7, d7, d2 \n\t" 1287020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1288020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // shift the 128 bit registers, containing the 16 1289020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // bit scaled values back to 8 bits, narrowing the 1290020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // results to 64 bit registers. 1291020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d8, q4, #8 \n\t" 1292020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d9, q5, #8 \n\t" 1293020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d10, q6, #8 \n\t" 1294020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vshrn.i16 d11, q7, #8 \n\t" 1295020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1296020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // adding back the color, using 128 bit registers. 1297020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vadd.i8 q6, q4, q0 \n\t" 1298020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vadd.i8 q7, q5, q0 \n\t" 1299020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1300020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // store back the 8 calculated pixels (2 128 bit 1301020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // registers), and increment dst. 1302020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "vst1.32 {d12-d15}, [%[dst]]! \n\t" 1303020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1304020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "subs %[count], %[count], #8 \n\t" 1305020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "bge Loop_Color32 \n\t" 1306020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [src] "+r" (src), [dst] "+r" (dst), [count] "+r" (count) 1307020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : [color] "r" (color), [scale] "r" (scale) 1308020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com : "cc", "memory", 1309020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", 1310020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15" 1311020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com ); 1312020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // At this point, if we went through the inline assembly, count is 1313020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // a negative value: 1314020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // if the value is -8, there is no pixel left to process. 1315020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // if the value is -7, there is one pixel left to process 1316020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // ... 1317020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // And'ing it with 7 will give us the number of pixels 1318020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // left to process. 1319020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com count = count & 0x7; 1320020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1321020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1322020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com while (count > 0) { 1323020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com *dst = color + SkAlphaMulQ(*src, scale); 1324020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com src += 1; 1325020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com dst += 1; 1326020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com count--; 1327020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1328020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com } 1329020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com} 1330020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1331020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com/////////////////////////////////////////////////////////////////////////////// 1332020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1333020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comconst SkBlitRow::Proc sk_blitrow_platform_565_procs_arm_neon[] = { 1334020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // no dither 1335020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // NOTE: For the two functions below, we don't have a special version 1336020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // that assumes that each source pixel is opaque. But our S32A is 1337020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // still faster than the default, so use it. 1338020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32A_D565_Opaque_neon, // really S32_D565_Opaque 1339020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32A_D565_Blend_neon, // really S32_D565_Blend 1340020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32A_D565_Opaque_neon, 1341020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32A_D565_Blend_neon, 1342020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1343020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com // dither 1344020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32_D565_Opaque_Dither_neon, 1345020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32_D565_Blend_Dither_neon, 1346020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com S32A_D565_Opaque_Dither_neon, 1347020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com NULL, // S32A_D565_Blend_Dither 1348020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com}; 1349020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com 1350020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.comconst SkBlitRow::Proc32 sk_blitrow_platform_32_procs_arm_neon[] = { 1351020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com NULL, // S32_Opaque, 1352935e9f4fafdfc64130e6be9ea2bb30e3bafd852armistry@google.com S32_Blend_BlitRow32_neon, // S32_Blend, 13539fba280106226e22454cb36bc90aa37badd37976djsollen@google.com /* 13549fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * We have two choices for S32A_Opaque procs. The one reads the src alpha 13559fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * value and attempts to optimize accordingly. The optimization is 13569fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * sensitive to the source content and is not a win in all cases. For 13579fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * example, if there are a lot of transitions between the alpha states, 13589fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * the performance will almost certainly be worse. However, for many 13599fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * common cases the performance is equivalent or better than the standard 13609fba280106226e22454cb36bc90aa37badd37976djsollen@google.com * case where we do not inspect the src alpha. 13619fba280106226e22454cb36bc90aa37badd37976djsollen@google.com */ 13629fba280106226e22454cb36bc90aa37badd37976djsollen@google.com#if SK_A32_SHIFT == 24 13639fba280106226e22454cb36bc90aa37badd37976djsollen@google.com // This proc assumes the alpha value occupies bits 24-32 of each SkPMColor 13649fba280106226e22454cb36bc90aa37badd37976djsollen@google.com S32A_Opaque_BlitRow32_neon_src_alpha, // S32A_Opaque, 13659fba280106226e22454cb36bc90aa37badd37976djsollen@google.com#else 13669fba280106226e22454cb36bc90aa37badd37976djsollen@google.com S32A_Opaque_BlitRow32_neon, // S32A_Opaque, 13679fba280106226e22454cb36bc90aa37badd37976djsollen@google.com#endif 1368c32b5699d35d14383226f147f965a79cce48c7abcommit-bot@chromium.org S32A_Blend_BlitRow32_neon // S32A_Blend 1369020093fc7c050ca7967c1d358ca14f7f6379dd69digit@google.com}; 1370