xref: /external/skia/src/opts/SkBlitRow_opts_arm_neon.cpp

/*
 * Copyright 2012 The Android Open Source Project
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "SkBlitRow_opts_arm_neon.h"

#include "SkBlitMask.h"
#include "SkBlitRow.h"
#include "SkColorPriv.h"
#include "SkDither.h"
#include "SkMathPriv.h"
#include "SkUtils.h"

#include "SkColor_opts_neon.h"
#include <arm_neon.h>

#ifdef SK_CPU_ARM64
static inline uint8x8x4_t sk_vld4_u8_arm64_3(const SkPMColor* SK_RESTRICT & src) {
    uint8x8x4_t vsrc;
    uint8x8_t vsrc_0, vsrc_1, vsrc_2;

    asm (
        "ld4    {v0.8b - v3.8b}, [%[src]], #32 \t\n"
        "mov    %[vsrc0].8b, v0.8b             \t\n"
        "mov    %[vsrc1].8b, v1.8b             \t\n"
        "mov    %[vsrc2].8b, v2.8b             \t\n"
        : [vsrc0] "=w" (vsrc_0), [vsrc1] "=w" (vsrc_1),
          [vsrc2] "=w" (vsrc_2), [src] "+&r" (src)
        : : "v0", "v1", "v2", "v3"
    );

    vsrc.val[0] = vsrc_0;
    vsrc.val[1] = vsrc_1;
    vsrc.val[2] = vsrc_2;

    return vsrc;
}

static inline uint8x8x4_t sk_vld4_u8_arm64_4(const SkPMColor* SK_RESTRICT & src) {
    uint8x8x4_t vsrc;
    uint8x8_t vsrc_0, vsrc_1, vsrc_2, vsrc_3;

    asm (
        "ld4    {v0.8b - v3.8b}, [%[src]], #32 \t\n"
        "mov    %[vsrc0].8b, v0.8b             \t\n"
        "mov    %[vsrc1].8b, v1.8b             \t\n"
        "mov    %[vsrc2].8b, v2.8b             \t\n"
        "mov    %[vsrc3].8b, v3.8b             \t\n"
        : [vsrc0] "=w" (vsrc_0), [vsrc1] "=w" (vsrc_1),
          [vsrc2] "=w" (vsrc_2), [vsrc3] "=w" (vsrc_3),
          [src] "+&r" (src)
        : : "v0", "v1", "v2", "v3"
    );

    vsrc.val[0] = vsrc_0;
    vsrc.val[1] = vsrc_1;
    vsrc.val[2] = vsrc_2;
    vsrc.val[3] = vsrc_3;

    return vsrc;
}
#endif

void S32_D565_Opaque_neon(uint16_t* SK_RESTRICT dst,
                           const SkPMColor* SK_RESTRICT src, int count,
                           U8CPU alpha, int /*x*/, int /*y*/) {
    SkASSERT(255 == alpha);

    while (count >= 8) {
        uint8x8x4_t vsrc;
        uint16x8_t vdst;

        // Load
#ifdef SK_CPU_ARM64
        vsrc = sk_vld4_u8_arm64_3(src);
#else
        vsrc = vld4_u8((uint8_t*)src);
        src += 8;
#endif

        // Convert src to 565
        vdst = SkPixel32ToPixel16_neon8(vsrc);

        // Store
        vst1q_u16(dst, vdst);

        // Prepare next iteration
        dst += 8;
        count -= 8;
    };

    // Leftovers
    while (count > 0) {
        SkPMColor c = *src++;
        SkPMColorAssert(c);
        *dst = SkPixel32ToPixel16_ToU16(c);
        dst++;
        count--;
    };
}

void S32_D565_Blend_neon(uint16_t* SK_RESTRICT dst,
                          const SkPMColor* SK_RESTRICT src, int count,
                          U8CPU alpha, int /*x*/, int /*y*/) {
    SkASSERT(255 > alpha);

    uint16x8_t vmask_blue, vscale;

    // prepare constants
    vscale = vdupq_n_u16(SkAlpha255To256(alpha));
    vmask_blue = vmovq_n_u16(0x1F);

    while (count >= 8) {
        uint8x8x4_t vsrc;
        uint16x8_t vdst, vdst_r, vdst_g, vdst_b;
        uint16x8_t vres_r, vres_g, vres_b;

        // Load src
#ifdef SK_CPU_ARM64
        vsrc = sk_vld4_u8_arm64_3(src);
#else
        {
        register uint8x8_t d0 asm("d0");
        register uint8x8_t d1 asm("d1");
        register uint8x8_t d2 asm("d2");
        register uint8x8_t d3 asm("d3");

        asm (
            "vld4.8    {d0-d3},[%[src]]!"
            : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+&r" (src)
            :
        );
        vsrc.val[0] = d0;
        vsrc.val[1] = d1;
        vsrc.val[2] = d2;
        }
#endif

        // Load and unpack dst
        vdst = vld1q_u16(dst);
        vdst_g = vshlq_n_u16(vdst, 5);        // shift green to top of lanes
        vdst_b = vandq_u16(vdst, vmask_blue); // extract blue
        vdst_r = vshrq_n_u16(vdst, 6+5);      // extract red
        vdst_g = vshrq_n_u16(vdst_g, 5+5);    // extract green

        // Shift src to 565 range
        vsrc.val[NEON_R] = vshr_n_u8(vsrc.val[NEON_R], 3);
        vsrc.val[NEON_G] = vshr_n_u8(vsrc.val[NEON_G], 2);
        vsrc.val[NEON_B] = vshr_n_u8(vsrc.val[NEON_B], 3);

        // Scale src - dst
        vres_r = vmovl_u8(vsrc.val[NEON_R]) - vdst_r;
        vres_g = vmovl_u8(vsrc.val[NEON_G]) - vdst_g;
        vres_b = vmovl_u8(vsrc.val[NEON_B]) - vdst_b;

        vres_r = vshrq_n_u16(vres_r * vscale, 8);
        vres_g = vshrq_n_u16(vres_g * vscale, 8);
        vres_b = vshrq_n_u16(vres_b * vscale, 8);

        vres_r += vdst_r;
        vres_g += vdst_g;
        vres_b += vdst_b;

        // Combine
        vres_b = vsliq_n_u16(vres_b, vres_g, 5);    // insert green into blue
        vres_b = vsliq_n_u16(vres_b, vres_r, 6+5);  // insert red into green/blue

        // Store
        vst1q_u16(dst, vres_b);
        dst += 8;
        count -= 8;
    }
    if (count > 0) {
        int scale = SkAlpha255To256(alpha);
        do {
            SkPMColor c = *src++;
            SkPMColorAssert(c);
            uint16_t d = *dst;
            *dst++ = SkPackRGB16(
                    SkAlphaBlend(SkPacked32ToR16(c), SkGetPackedR16(d), scale),
                    SkAlphaBlend(SkPacked32ToG16(c), SkGetPackedG16(d), scale),
                    SkAlphaBlend(SkPacked32ToB16(c), SkGetPackedB16(d), scale));
        } while (--count != 0);
    }
}

#ifdef SK_CPU_ARM32
void S32A_D565_Opaque_neon(uint16_t* SK_RESTRICT dst,
                           const SkPMColor* SK_RESTRICT src, int count,
                           U8CPU alpha, int /*x*/, int /*y*/) {
    SkASSERT(255 == alpha);

    if (count >= 8) {
        uint16_t* SK_RESTRICT keep_dst = 0;

        asm volatile (
                      "ands       ip, %[count], #7            \n\t"
                      "vmov.u8    d31, #1<<7                  \n\t"
                      "vld1.16    {q12}, [%[dst]]             \n\t"
                      "vld4.8     {d0-d3}, [%[src]]           \n\t"
                      // Thumb does not support the standard ARM conditional
                      // instructions but instead requires the 'it' instruction
                      // to signal conditional execution
                      "it eq                                  \n\t"
                      "moveq      ip, #8                      \n\t"
                      "mov        %[keep_dst], %[dst]         \n\t"

                      "add        %[src], %[src], ip, LSL#2   \n\t"
                      "add        %[dst], %[dst], ip, LSL#1   \n\t"
                      "subs       %[count], %[count], ip      \n\t"
                      "b          9f                          \n\t"
                      // LOOP
                      "2:                                         \n\t"

                      "vld1.16    {q12}, [%[dst]]!            \n\t"
                      "vld4.8     {d0-d3}, [%[src]]!          \n\t"
                      "vst1.16    {q10}, [%[keep_dst]]        \n\t"
                      "sub        %[keep_dst], %[dst], #8*2   \n\t"
                      "subs       %[count], %[count], #8      \n\t"
                      "9:                                         \n\t"
                      "pld        [%[dst],#32]                \n\t"
                      // expand 0565 q12 to 8888 {d4-d7}
                      "vmovn.u16  d4, q12                     \n\t"
                      "vshr.u16   q11, q12, #5                \n\t"
                      "vshr.u16   q10, q12, #6+5              \n\t"
                      "vmovn.u16  d5, q11                     \n\t"
                      "vmovn.u16  d6, q10                     \n\t"
                      "vshl.u8    d4, d4, #3                  \n\t"
                      "vshl.u8    d5, d5, #2                  \n\t"
                      "vshl.u8    d6, d6, #3                  \n\t"

                      "vmovl.u8   q14, d31                    \n\t"
                      "vmovl.u8   q13, d31                    \n\t"
                      "vmovl.u8   q12, d31                    \n\t"

                      // duplicate in 4/2/1 & 8pix vsns
                      "vmvn.8     d30, d3                     \n\t"
                      "vmlal.u8   q14, d30, d6                \n\t"
                      "vmlal.u8   q13, d30, d5                \n\t"
                      "vmlal.u8   q12, d30, d4                \n\t"
                      "vshr.u16   q8, q14, #5                 \n\t"
                      "vshr.u16   q9, q13, #6                 \n\t"
                      "vaddhn.u16 d6, q14, q8                 \n\t"
                      "vshr.u16   q8, q12, #5                 \n\t"
                      "vaddhn.u16 d5, q13, q9                 \n\t"
                      "vaddhn.u16 d4, q12, q8                 \n\t"
                      // intentionally don't calculate alpha
                      // result in d4-d6

            #ifdef SK_PMCOLOR_IS_RGBA
                      "vqadd.u8   d6, d6, d0                  \n\t"
                      "vqadd.u8   d5, d5, d1                  \n\t"
                      "vqadd.u8   d4, d4, d2                  \n\t"
            #else
                      "vqadd.u8   d6, d6, d2                  \n\t"
                      "vqadd.u8   d5, d5, d1                  \n\t"
                      "vqadd.u8   d4, d4, d0                  \n\t"
            #endif

                      // pack 8888 {d4-d6} to 0565 q10
                      "vshll.u8   q10, d6, #8                 \n\t"
                      "vshll.u8   q3, d5, #8                  \n\t"
                      "vshll.u8   q2, d4, #8                  \n\t"
                      "vsri.u16   q10, q3, #5                 \n\t"
                      "vsri.u16   q10, q2, #11                \n\t"

                      "bne        2b                          \n\t"

                      "1:                                         \n\t"
                      "vst1.16      {q10}, [%[keep_dst]]      \n\t"
                      : [count] "+r" (count)
                      : [dst] "r" (dst), [keep_dst] "r" (keep_dst), [src] "r" (src)
                      : "ip", "cc", "memory", "d0","d1","d2","d3","d4","d5","d6","d7",
                      "d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29",
                      "d30","d31"
                      );
    }
    else
    {   // handle count < 8
        uint16_t* SK_RESTRICT keep_dst = 0;

        asm volatile (
                      "vmov.u8    d31, #1<<7                  \n\t"
                      "mov        %[keep_dst], %[dst]         \n\t"

                      "tst        %[count], #4                \n\t"
                      "beq        14f                         \n\t"
                      "vld1.16    {d25}, [%[dst]]!            \n\t"
                      "vld1.32    {q1}, [%[src]]!             \n\t"

                      "14:                                        \n\t"
                      "tst        %[count], #2                \n\t"
                      "beq        12f                         \n\t"
                      "vld1.32    {d24[1]}, [%[dst]]!         \n\t"
                      "vld1.32    {d1}, [%[src]]!             \n\t"

                      "12:                                        \n\t"
                      "tst        %[count], #1                \n\t"
                      "beq        11f                         \n\t"
                      "vld1.16    {d24[1]}, [%[dst]]!         \n\t"
                      "vld1.32    {d0[1]}, [%[src]]!          \n\t"

                      "11:                                        \n\t"
                      // unzips achieve the same as a vld4 operation
                      "vuzp.u16   q0, q1                      \n\t"
                      "vuzp.u8    d0, d1                      \n\t"
                      "vuzp.u8    d2, d3                      \n\t"
                      // expand 0565 q12 to 8888 {d4-d7}
                      "vmovn.u16  d4, q12                     \n\t"
                      "vshr.u16   q11, q12, #5                \n\t"
                      "vshr.u16   q10, q12, #6+5              \n\t"
                      "vmovn.u16  d5, q11                     \n\t"
                      "vmovn.u16  d6, q10                     \n\t"
                      "vshl.u8    d4, d4, #3                  \n\t"
                      "vshl.u8    d5, d5, #2                  \n\t"
                      "vshl.u8    d6, d6, #3                  \n\t"

                      "vmovl.u8   q14, d31                    \n\t"
                      "vmovl.u8   q13, d31                    \n\t"
                      "vmovl.u8   q12, d31                    \n\t"

                      // duplicate in 4/2/1 & 8pix vsns
                      "vmvn.8     d30, d3                     \n\t"
                      "vmlal.u8   q14, d30, d6                \n\t"
                      "vmlal.u8   q13, d30, d5                \n\t"
                      "vmlal.u8   q12, d30, d4                \n\t"
                      "vshr.u16   q8, q14, #5                 \n\t"
                      "vshr.u16   q9, q13, #6                 \n\t"
                      "vaddhn.u16 d6, q14, q8                 \n\t"
                      "vshr.u16   q8, q12, #5                 \n\t"
                      "vaddhn.u16 d5, q13, q9                 \n\t"
                      "vaddhn.u16 d4, q12, q8                 \n\t"
                      // intentionally don't calculate alpha
                      // result in d4-d6

            #ifdef SK_PMCOLOR_IS_RGBA
                      "vqadd.u8   d6, d6, d0                  \n\t"
                      "vqadd.u8   d5, d5, d1                  \n\t"
                      "vqadd.u8   d4, d4, d2                  \n\t"
            #else
                      "vqadd.u8   d6, d6, d2                  \n\t"
                      "vqadd.u8   d5, d5, d1                  \n\t"
                      "vqadd.u8   d4, d4, d0                  \n\t"
            #endif

                      // pack 8888 {d4-d6} to 0565 q10
                      "vshll.u8   q10, d6, #8                 \n\t"
                      "vshll.u8   q3, d5, #8                  \n\t"
                      "vshll.u8   q2, d4, #8                  \n\t"
                      "vsri.u16   q10, q3, #5                 \n\t"
                      "vsri.u16   q10, q2, #11                \n\t"

                      // store
                      "tst        %[count], #4                \n\t"
                      "beq        24f                         \n\t"
                      "vst1.16    {d21}, [%[keep_dst]]!       \n\t"

                      "24:                                        \n\t"
                      "tst        %[count], #2                \n\t"
                      "beq        22f                         \n\t"
                      "vst1.32    {d20[1]}, [%[keep_dst]]!    \n\t"

                      "22:                                        \n\t"
                      "tst        %[count], #1                \n\t"
                      "beq        21f                         \n\t"
                      "vst1.16    {d20[1]}, [%[keep_dst]]!    \n\t"

                      "21:                                        \n\t"
                      : [count] "+r" (count)
                      : [dst] "r" (dst), [keep_dst] "r" (keep_dst), [src] "r" (src)
                      : "ip", "cc", "memory", "d0","d1","d2","d3","d4","d5","d6","d7",
                      "d16","d17","d18","d19","d20","d21","d22","d23","d24","d25","d26","d27","d28","d29",
                      "d30","d31"
                      );
    }
}

#else // #ifdef SK_CPU_ARM32

void S32A_D565_Opaque_neon(uint16_t* SK_RESTRICT dst,
                           const SkPMColor* SK_RESTRICT src, int count,
                           U8CPU alpha, int /*x*/, int /*y*/) {
    SkASSERT(255 == alpha);

    if (count >= 16) {
        asm (
            "movi    v4.8h, #0x80                   \t\n"

            "1:                                     \t\n"
            "sub     %w[count], %w[count], #16      \t\n"
            "ld1     {v16.8h-v17.8h}, [%[dst]]      \t\n"
            "ld4     {v0.16b-v3.16b}, [%[src]], #64 \t\n"
            "prfm    pldl1keep, [%[src],#512]       \t\n"
            "prfm    pldl1keep, [%[dst],#256]       \t\n"
            "ushr    v20.8h, v17.8h, #5             \t\n"
            "ushr    v31.8h, v16.8h, #5             \t\n"
            "xtn     v6.8b, v31.8h                  \t\n"
            "xtn2    v6.16b, v20.8h                 \t\n"
            "ushr    v20.8h, v17.8h, #11            \t\n"
            "shl     v19.16b, v6.16b, #2            \t\n"
            "ushr    v31.8h, v16.8h, #11            \t\n"
            "xtn     v22.8b, v31.8h                 \t\n"
            "xtn2    v22.16b, v20.8h                \t\n"
            "shl     v18.16b, v22.16b, #3           \t\n"
            "mvn     v3.16b, v3.16b                 \t\n"
            "xtn     v16.8b, v16.8h                 \t\n"
            "mov     v7.16b, v4.16b                 \t\n"
            "xtn2    v16.16b, v17.8h                \t\n"
            "umlal   v7.8h, v3.8b, v19.8b           \t\n"
            "shl     v16.16b, v16.16b, #3           \t\n"
            "mov     v22.16b, v4.16b                \t\n"
            "ushr    v24.8h, v7.8h, #6              \t\n"
            "umlal   v22.8h, v3.8b, v18.8b          \t\n"
            "ushr    v20.8h, v22.8h, #5             \t\n"
            "addhn   v20.8b, v22.8h, v20.8h         \t\n"
            "cmp     %w[count], #16                 \t\n"
            "mov     v6.16b, v4.16b                 \t\n"
            "mov     v5.16b, v4.16b                 \t\n"
            "umlal   v6.8h, v3.8b, v16.8b           \t\n"
            "umlal2  v5.8h, v3.16b, v19.16b         \t\n"
            "mov     v17.16b, v4.16b                \t\n"
            "ushr    v19.8h, v6.8h, #5              \t\n"
            "umlal2  v17.8h, v3.16b, v18.16b        \t\n"
            "addhn   v7.8b, v7.8h, v24.8h           \t\n"
            "ushr    v18.8h, v5.8h, #6              \t\n"
            "ushr    v21.8h, v17.8h, #5             \t\n"
            "addhn2  v7.16b, v5.8h, v18.8h          \t\n"
            "addhn2  v20.16b, v17.8h, v21.8h        \t\n"
            "mov     v22.16b, v4.16b                \t\n"
            "addhn   v6.8b, v6.8h, v19.8h           \t\n"
            "umlal2  v22.8h, v3.16b, v16.16b        \t\n"
            "ushr    v5.8h, v22.8h, #5              \t\n"
            "addhn2  v6.16b, v22.8h, v5.8h          \t\n"
            "uqadd   v7.16b, v1.16b, v7.16b         \t\n"
#if SK_PMCOLOR_BYTE_ORDER(B,G,R,A)
            "uqadd   v20.16b, v2.16b, v20.16b       \t\n"
            "uqadd   v6.16b, v0.16b, v6.16b         \t\n"
#elif SK_PMCOLOR_BYTE_ORDER(R,G,B,A)
            "uqadd   v20.16b, v0.16b, v20.16b       \t\n"
            "uqadd   v6.16b, v2.16b, v6.16b         \t\n"
#else
#error "This function only supports BGRA and RGBA."
#endif
            "shll    v22.8h, v20.8b, #8             \t\n"
            "shll    v5.8h, v7.8b, #8               \t\n"
            "sri     v22.8h, v5.8h, #5              \t\n"
            "shll    v17.8h, v6.8b, #8              \t\n"
            "shll2   v23.8h, v20.16b, #8            \t\n"
            "shll2   v7.8h, v7.16b, #8              \t\n"
            "sri     v22.8h, v17.8h, #11            \t\n"
            "sri     v23.8h, v7.8h, #5              \t\n"
            "shll2   v6.8h, v6.16b, #8              \t\n"
            "st1     {v22.8h}, [%[dst]], #16        \t\n"
            "sri     v23.8h, v6.8h, #11             \t\n"
            "st1     {v23.8h}, [%[dst]], #16        \t\n"
            "b.ge    1b                             \t\n"
            : [dst] "+&r" (dst), [src] "+&r" (src), [count] "+&r" (count)
            :: "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
               "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
               "v31"
        );
    }
        // Leftovers
    if (count > 0) {
        do {
            SkPMColor c = *src++;
            SkPMColorAssert(c);
            if (c) {
                *dst = SkSrcOver32To16(c, *dst);
            }
            dst += 1;
        } while (--count != 0);
    }
}
#endif // #ifdef SK_CPU_ARM32

static uint32_t pmcolor_to_expand16(SkPMColor c) {
    unsigned r = SkGetPackedR32(c);
    unsigned g = SkGetPackedG32(c);
    unsigned b = SkGetPackedB32(c);
    return (g << 24) | (r << 13) | (b << 2);
}

void Color32A_D565_neon(uint16_t dst[], SkPMColor src, int count, int x, int y) {
    uint32_t src_expand;
    unsigned scale;
    uint16x8_t vmask_blue;

    if (count <= 0) return;
    SkASSERT(((size_t)dst & 0x01) == 0);

    /*
     * This preamble code is in order to make dst aligned to 8 bytes
     * in the next mutiple bytes read & write access.
     */
    src_expand = pmcolor_to_expand16(src);
    scale = SkAlpha255To256(0xFF - SkGetPackedA32(src)) >> 3;

#define DST_ALIGN 8

    /*
     * preamble_size is in byte, meantime, this blend32_16_row_neon updates 2 bytes at a time.
     */
    int preamble_size = (DST_ALIGN - (size_t)dst) & (DST_ALIGN - 1);

    for (int i = 0; i < preamble_size; i+=2, dst++) {
        uint32_t dst_expand = SkExpand_rgb_16(*dst) * scale;
        *dst = SkCompact_rgb_16((src_expand + dst_expand) >> 5);
        if (--count == 0)
            break;
    }

    int count16 = 0;
    count16 = count >> 4;
    vmask_blue = vmovq_n_u16(SK_B16_MASK);

    if (count16) {
        uint16x8_t wide_sr;
        uint16x8_t wide_sg;
        uint16x8_t wide_sb;
        uint16x8_t wide_256_sa;

        unsigned sr = SkGetPackedR32(src);
        unsigned sg = SkGetPackedG32(src);
        unsigned sb = SkGetPackedB32(src);
        unsigned sa = SkGetPackedA32(src);

        // Operation: dst_rgb = src_rgb + ((256 - src_a) >> 3) x dst_rgb
        // sr: 8-bit based, dr: 5-bit based, with dr x ((256-sa)>>3), 5-bit left shifted,
        //thus, for sr, do 2-bit left shift to match MSB : (8 + 2 = 5 + 5)
        wide_sr = vshlq_n_u16(vmovl_u8(vdup_n_u8(sr)), 2); // widen and src_red shift

        // sg: 8-bit based, dg: 6-bit based, with dg x ((256-sa)>>3), 5-bit left shifted,
        //thus, for sg, do 3-bit left shift to match MSB : (8 + 3 = 6 + 5)
        wide_sg = vshlq_n_u16(vmovl_u8(vdup_n_u8(sg)), 3); // widen and src_grn shift

        // sb: 8-bit based, db: 5-bit based, with db x ((256-sa)>>3), 5-bit left shifted,
        //thus, for sb, do 2-bit left shift to match MSB : (8 + 2 = 5 + 5)
        wide_sb = vshlq_n_u16(vmovl_u8(vdup_n_u8(sb)), 2); // widen and src blu shift

        wide_256_sa =
            vshrq_n_u16(vsubw_u8(vdupq_n_u16(256), vdup_n_u8(sa)), 3); // (256 - sa) >> 3

        while (count16-- > 0) {
            uint16x8_t vdst1, vdst1_r, vdst1_g, vdst1_b;
            uint16x8_t vdst2, vdst2_r, vdst2_g, vdst2_b;
            vdst1 = vld1q_u16(dst);
            dst += 8;
            vdst2 = vld1q_u16(dst);
            dst -= 8;    //to store dst again.

            vdst1_g = vshlq_n_u16(vdst1, SK_R16_BITS);                 // shift green to top of lanes
            vdst1_b = vdst1 & vmask_blue;                              // extract blue
            vdst1_r = vshrq_n_u16(vdst1, SK_R16_SHIFT);                // extract red
            vdst1_g = vshrq_n_u16(vdst1_g, SK_R16_BITS + SK_B16_BITS); // extract green

            vdst2_g = vshlq_n_u16(vdst2, SK_R16_BITS);                 // shift green to top of lanes
            vdst2_b = vdst2 & vmask_blue;                              // extract blue
            vdst2_r = vshrq_n_u16(vdst2, SK_R16_SHIFT);                // extract red
            vdst2_g = vshrq_n_u16(vdst2_g, SK_R16_BITS + SK_B16_BITS); // extract green

            vdst1_r = vmlaq_u16(wide_sr, wide_256_sa, vdst1_r);        // sr + (256-sa) x dr1
            vdst1_g = vmlaq_u16(wide_sg, wide_256_sa, vdst1_g);        // sg + (256-sa) x dg1
            vdst1_b = vmlaq_u16(wide_sb, wide_256_sa, vdst1_b);        // sb + (256-sa) x db1

            vdst2_r = vmlaq_u16(wide_sr, wide_256_sa, vdst2_r);        // sr + (256-sa) x dr2
            vdst2_g = vmlaq_u16(wide_sg, wide_256_sa, vdst2_g);        // sg + (256-sa) x dg2
            vdst2_b = vmlaq_u16(wide_sb, wide_256_sa, vdst2_b);        // sb + (256-sa) x db2

            vdst1_r = vshrq_n_u16(vdst1_r, 5);                         // 5-bit right shift for 5-bit red
            vdst1_g = vshrq_n_u16(vdst1_g, 5);                         // 5-bit right shift for 6-bit green
            vdst1_b = vshrq_n_u16(vdst1_b, 5);                         // 5-bit right shift for 5-bit blue

            vdst1 = vsliq_n_u16(vdst1_b, vdst1_g, SK_G16_SHIFT);       // insert green into blue
            vdst1 = vsliq_n_u16(vdst1, vdst1_r, SK_R16_SHIFT);         // insert red into green/blue

            vdst2_r = vshrq_n_u16(vdst2_r, 5);                         // 5-bit right shift for 5-bit red
            vdst2_g = vshrq_n_u16(vdst2_g, 5);                         // 5-bit right shift for 6-bit green
            vdst2_b = vshrq_n_u16(vdst2_b, 5);                         // 5-bit right shift for 5-bit blue

            vdst2 = vsliq_n_u16(vdst2_b, vdst2_g, SK_G16_SHIFT);       // insert green into blue
            vdst2 = vsliq_n_u16(vdst2, vdst2_r, SK_R16_SHIFT);         // insert red into green/blue

            vst1q_u16(dst, vdst1);
            dst += 8;
            vst1q_u16(dst, vdst2);
            dst += 8;
        }
    }

    count &= 0xF;
    if (count > 0) {
        do {
            uint32_t dst_expand = SkExpand_rgb_16(*dst) * scale;
            *dst = SkCompact_rgb_16((src_expand + dst_expand) >> 5);
            dst += 1;
        } while (--count != 0);
    }
}

static inline uint16x8_t SkDiv255Round_neon8(uint16x8_t prod) {
    prod += vdupq_n_u16(128);
    prod += vshrq_n_u16(prod, 8);
    return vshrq_n_u16(prod, 8);
}

void S32A_D565_Blend_neon(uint16_t* SK_RESTRICT dst,
                          const SkPMColor* SK_RESTRICT src, int count,
                          U8CPU alpha, int /*x*/, int /*y*/) {
   SkASSERT(255 > alpha);

    /* This code implements a Neon version of S32A_D565_Blend. The results have
     * a few mismatches compared to the original code. These mismatches never
     * exceed 1.
     */

    if (count >= 8) {
        uint16x8_t valpha_max, vmask_blue;
        uint8x8_t valpha;

        // prepare constants
        valpha_max = vmovq_n_u16(255);
        valpha = vdup_n_u8(alpha);
        vmask_blue = vmovq_n_u16(SK_B16_MASK);

        do {
            uint16x8_t vdst, vdst_r, vdst_g, vdst_b;
            uint16x8_t vres_a, vres_r, vres_g, vres_b;
            uint8x8x4_t vsrc;

            // load pixels
            vdst = vld1q_u16(dst);
#ifdef SK_CPU_ARM64
            vsrc = sk_vld4_u8_arm64_4(src);
#elif (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 6))
            asm (
                "vld4.u8 %h[vsrc], [%[src]]!"
                : [vsrc] "=w" (vsrc), [src] "+&r" (src)
                : :
            );
#else
            register uint8x8_t d0 asm("d0");
            register uint8x8_t d1 asm("d1");
            register uint8x8_t d2 asm("d2");
            register uint8x8_t d3 asm("d3");

            asm volatile (
                "vld4.u8    {d0-d3},[%[src]]!;"
                : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3),
                  [src] "+&r" (src)
                : :
            );
            vsrc.val[0] = d0;
            vsrc.val[1] = d1;
            vsrc.val[2] = d2;
            vsrc.val[3] = d3;
#endif


            // deinterleave dst
            vdst_g = vshlq_n_u16(vdst, SK_R16_BITS);        // shift green to top of lanes
            vdst_b = vdst & vmask_blue;                     // extract blue
            vdst_r = vshrq_n_u16(vdst, SK_R16_SHIFT);       // extract red
            vdst_g = vshrq_n_u16(vdst_g, SK_R16_BITS + SK_B16_BITS); // extract green

            // shift src to 565
            vsrc.val[NEON_R] = vshr_n_u8(vsrc.val[NEON_R], 8 - SK_R16_BITS);
            vsrc.val[NEON_G] = vshr_n_u8(vsrc.val[NEON_G], 8 - SK_G16_BITS);
            vsrc.val[NEON_B] = vshr_n_u8(vsrc.val[NEON_B], 8 - SK_B16_BITS);

            // calc src * src_scale
            vres_a = vmull_u8(vsrc.val[NEON_A], valpha);
            vres_r = vmull_u8(vsrc.val[NEON_R], valpha);
            vres_g = vmull_u8(vsrc.val[NEON_G], valpha);
            vres_b = vmull_u8(vsrc.val[NEON_B], valpha);

            // prepare dst_scale
            vres_a = SkDiv255Round_neon8(vres_a);
            vres_a = valpha_max - vres_a; // 255 - (sa * src_scale) / 255

            // add dst * dst_scale to previous result
            vres_r = vmlaq_u16(vres_r, vdst_r, vres_a);
            vres_g = vmlaq_u16(vres_g, vdst_g, vres_a);
            vres_b = vmlaq_u16(vres_b, vdst_b, vres_a);

#ifdef S32A_D565_BLEND_EXACT
            // It is possible to get exact results with this but it is slow,
            // even slower than C code in some cases
            vres_r = SkDiv255Round_neon8(vres_r);
            vres_g = SkDiv255Round_neon8(vres_g);
            vres_b = SkDiv255Round_neon8(vres_b);
#else
            vres_r = vrshrq_n_u16(vres_r, 8);
            vres_g = vrshrq_n_u16(vres_g, 8);
            vres_b = vrshrq_n_u16(vres_b, 8);
#endif
            // pack result
            vres_b = vsliq_n_u16(vres_b, vres_g, SK_G16_SHIFT); // insert green into blue
            vres_b = vsliq_n_u16(vres_b, vres_r, SK_R16_SHIFT); // insert red into green/blue

            // store
            vst1q_u16(dst, vres_b);
            dst += 8;
            count -= 8;
        } while (count >= 8);
    }

    // leftovers
    while (count-- > 0) {
        SkPMColor sc = *src++;
        if (sc) {
            uint16_t dc = *dst;
            unsigned dst_scale = 255 - SkMulDiv255Round(SkGetPackedA32(sc), alpha);
            unsigned dr = (SkPacked32ToR16(sc) * alpha) + (SkGetPackedR16(dc) * dst_scale);
            unsigned dg = (SkPacked32ToG16(sc) * alpha) + (SkGetPackedG16(dc) * dst_scale);
            unsigned db = (SkPacked32ToB16(sc) * alpha) + (SkGetPackedB16(dc) * dst_scale);
            *dst = SkPackRGB16(SkDiv255Round(dr), SkDiv255Round(dg), SkDiv255Round(db));
        }
        dst += 1;
    }
}

/* dither matrix for Neon, derived from gDitherMatrix_3Bit_16.
 * each dither value is spaced out into byte lanes, and repeated
 * to allow an 8-byte load from offsets 0, 1, 2 or 3 from the
 * start of each row.
 */
static const uint8_t gDitherMatrix_Neon[48] = {
    0, 4, 1, 5, 0, 4, 1, 5, 0, 4, 1, 5,
    6, 2, 7, 3, 6, 2, 7, 3, 6, 2, 7, 3,
    1, 5, 0, 4, 1, 5, 0, 4, 1, 5, 0, 4,
    7, 3, 6, 2, 7, 3, 6, 2, 7, 3, 6, 2,

};

void S32_D565_Blend_Dither_neon(uint16_t *dst, const SkPMColor *src,
                                int count, U8CPU alpha, int x, int y)
{

    SkASSERT(255 > alpha);

    // rescale alpha to range 1 - 256
    int scale = SkAlpha255To256(alpha);

    if (count >= 8) {
        /* select row and offset for dither array */
        const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)];

        uint8x8_t vdither = vld1_u8(dstart);         // load dither values
        uint8x8_t vdither_g = vshr_n_u8(vdither, 1); // calc. green dither values

        int16x8_t vscale = vdupq_n_s16(scale);        // duplicate scale into neon reg
        uint16x8_t vmask_b = vdupq_n_u16(0x1F);         // set up blue mask

        do {

            uint8x8x4_t vsrc;
            uint8x8_t vsrc_r, vsrc_g, vsrc_b;
            uint8x8_t vsrc565_r, vsrc565_g, vsrc565_b;
            uint16x8_t vsrc_dit_r, vsrc_dit_g, vsrc_dit_b;
            uint16x8_t vsrc_res_r, vsrc_res_g, vsrc_res_b;
            uint16x8_t vdst;
            uint16x8_t vdst_r, vdst_g, vdst_b;
            int16x8_t vres_r, vres_g, vres_b;
            int8x8_t vres8_r, vres8_g, vres8_b;

            // Load source and add dither
#ifdef SK_CPU_ARM64
            vsrc = sk_vld4_u8_arm64_3(src);
#else
            {
            register uint8x8_t d0 asm("d0");
            register uint8x8_t d1 asm("d1");
            register uint8x8_t d2 asm("d2");
            register uint8x8_t d3 asm("d3");

            asm (
                "vld4.8    {d0-d3},[%[src]]! "
                : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+&r" (src)
                :
            );
            vsrc.val[0] = d0;
            vsrc.val[1] = d1;
            vsrc.val[2] = d2;
            }
#endif
            vsrc_r = vsrc.val[NEON_R];
            vsrc_g = vsrc.val[NEON_G];
            vsrc_b = vsrc.val[NEON_B];

            vsrc565_g = vshr_n_u8(vsrc_g, 6); // calc. green >> 6
            vsrc565_r = vshr_n_u8(vsrc_r, 5); // calc. red >> 5
            vsrc565_b = vshr_n_u8(vsrc_b, 5); // calc. blue >> 5

            vsrc_dit_g = vaddl_u8(vsrc_g, vdither_g); // add in dither to green and widen
            vsrc_dit_r = vaddl_u8(vsrc_r, vdither);   // add in dither to red and widen
            vsrc_dit_b = vaddl_u8(vsrc_b, vdither);   // add in dither to blue and widen

            vsrc_dit_r = vsubw_u8(vsrc_dit_r, vsrc565_r);  // sub shifted red from result
            vsrc_dit_g = vsubw_u8(vsrc_dit_g, vsrc565_g);  // sub shifted green from result
            vsrc_dit_b = vsubw_u8(vsrc_dit_b, vsrc565_b);  // sub shifted blue from result

            vsrc_res_r = vshrq_n_u16(vsrc_dit_r, 3);
            vsrc_res_g = vshrq_n_u16(vsrc_dit_g, 2);
            vsrc_res_b = vshrq_n_u16(vsrc_dit_b, 3);

            // Load dst and unpack
            vdst = vld1q_u16(dst);
            vdst_g = vshrq_n_u16(vdst, 5);                   // shift down to get green
            vdst_r = vshrq_n_u16(vshlq_n_u16(vdst, 5), 5+5); // double shift to extract red
            vdst_b = vandq_u16(vdst, vmask_b);               // mask to get blue

            // subtract dst from src and widen
            vres_r = vsubq_s16(vreinterpretq_s16_u16(vsrc_res_r), vreinterpretq_s16_u16(vdst_r));
            vres_g = vsubq_s16(vreinterpretq_s16_u16(vsrc_res_g), vreinterpretq_s16_u16(vdst_g));
            vres_b = vsubq_s16(vreinterpretq_s16_u16(vsrc_res_b), vreinterpretq_s16_u16(vdst_b));

            // multiply diffs by scale and shift
            vres_r = vmulq_s16(vres_r, vscale);
            vres_g = vmulq_s16(vres_g, vscale);
            vres_b = vmulq_s16(vres_b, vscale);

            vres8_r = vshrn_n_s16(vres_r, 8);
            vres8_g = vshrn_n_s16(vres_g, 8);
            vres8_b = vshrn_n_s16(vres_b, 8);

            // add dst to result
            vres_r = vaddw_s8(vreinterpretq_s16_u16(vdst_r), vres8_r);
            vres_g = vaddw_s8(vreinterpretq_s16_u16(vdst_g), vres8_g);
            vres_b = vaddw_s8(vreinterpretq_s16_u16(vdst_b), vres8_b);

            // put result into 565 format
            vres_b = vsliq_n_s16(vres_b, vres_g, 5);   // shift up green and insert into blue
            vres_b = vsliq_n_s16(vres_b, vres_r, 6+5); // shift up red and insert into blue

            // Store result
            vst1q_u16(dst, vreinterpretq_u16_s16(vres_b));

            // Next iteration
            dst += 8;
            count -= 8;

        } while (count >= 8);
    }

    // Leftovers
    if (count > 0) {
        int scale = SkAlpha255To256(alpha);
        DITHER_565_SCAN(y);
        do {
            SkPMColor c = *src++;
            SkPMColorAssert(c);

            int dither = DITHER_VALUE(x);
            int sr = SkGetPackedR32(c);
            int sg = SkGetPackedG32(c);
            int sb = SkGetPackedB32(c);
            sr = SkDITHER_R32To565(sr, dither);
            sg = SkDITHER_G32To565(sg, dither);
            sb = SkDITHER_B32To565(sb, dither);

            uint16_t d = *dst;
            *dst++ = SkPackRGB16(SkAlphaBlend(sr, SkGetPackedR16(d), scale),
                                 SkAlphaBlend(sg, SkGetPackedG16(d), scale),
                                 SkAlphaBlend(sb, SkGetPackedB16(d), scale));
            DITHER_INC_X(x);
        } while (--count != 0);
    }
}

/* Neon version of S32_Blend_BlitRow32()
 * portable version is in src/core/SkBlitRow_D32.cpp
 */
void S32_Blend_BlitRow32_neon(SkPMColor* SK_RESTRICT dst,
                              const SkPMColor* SK_RESTRICT src,
                              int count, U8CPU alpha) {
    SkASSERT(alpha <= 255);

    if (count <= 0) {
        return;
    }

    uint16_t src_scale = SkAlpha255To256(alpha);
    uint16_t dst_scale = 256 - src_scale;

    while (count >= 2) {
        uint8x8_t vsrc, vdst, vres;
        uint16x8_t vsrc_wide, vdst_wide;

        /* These commented prefetches are a big win for count
         * values > 64 on an A9 (Pandaboard) but hurt by 10% for count = 4.
         * They also hurt a little (<5%) on an A15
         */
        //__builtin_prefetch(src+32);
        //__builtin_prefetch(dst+32);

        // Load
        vsrc = vreinterpret_u8_u32(vld1_u32(src));
        vdst = vreinterpret_u8_u32(vld1_u32(dst));

        // Process src
        vsrc_wide = vmovl_u8(vsrc);
        vsrc_wide = vmulq_u16(vsrc_wide, vdupq_n_u16(src_scale));

        // Process dst
        vdst_wide = vmull_u8(vdst, vdup_n_u8(dst_scale));

        // Combine
        vdst_wide += vsrc_wide;
        vres = vshrn_n_u16(vdst_wide, 8);

        // Store
        vst1_u32(dst, vreinterpret_u32_u8(vres));

        src += 2;
        dst += 2;
        count -= 2;
    }

    if (count == 1) {
        uint8x8_t vsrc = vdup_n_u8(0), vdst = vdup_n_u8(0), vres;
        uint16x8_t vsrc_wide, vdst_wide;

        // Load
        vsrc = vreinterpret_u8_u32(vld1_lane_u32(src, vreinterpret_u32_u8(vsrc), 0));
        vdst = vreinterpret_u8_u32(vld1_lane_u32(dst, vreinterpret_u32_u8(vdst), 0));

        // Process
        vsrc_wide = vmovl_u8(vsrc);
        vsrc_wide = vmulq_u16(vsrc_wide, vdupq_n_u16(src_scale));
        vdst_wide = vmull_u8(vdst, vdup_n_u8(dst_scale));
        vdst_wide += vsrc_wide;
        vres = vshrn_n_u16(vdst_wide, 8);

        // Store
        vst1_lane_u32(dst, vreinterpret_u32_u8(vres), 0);
    }
}

#ifdef SK_CPU_ARM32
void S32A_Blend_BlitRow32_neon(SkPMColor* SK_RESTRICT dst,
                         const SkPMColor* SK_RESTRICT src,
                         int count, U8CPU alpha) {

    SkASSERT(255 > alpha);

    if (count <= 0) {
        return;
    }

    unsigned alpha256 = SkAlpha255To256(alpha);

    // First deal with odd counts
    if (count & 1) {
        uint8x8_t vsrc = vdup_n_u8(0), vdst = vdup_n_u8(0), vres;
        uint16x8_t vdst_wide, vsrc_wide;
        unsigned dst_scale;

        // Load
        vsrc = vreinterpret_u8_u32(vld1_lane_u32(src, vreinterpret_u32_u8(vsrc), 0));
        vdst = vreinterpret_u8_u32(vld1_lane_u32(dst, vreinterpret_u32_u8(vdst), 0));

        // Calc dst_scale
        dst_scale = vget_lane_u8(vsrc, 3);
        dst_scale = SkAlphaMulInv256(dst_scale, alpha256);

        // Process src
        vsrc_wide = vmovl_u8(vsrc);
        vsrc_wide = vmulq_n_u16(vsrc_wide, alpha256);

        // Process dst
        vdst_wide = vmovl_u8(vdst);
        vdst_wide = vmulq_n_u16(vdst_wide, dst_scale);

        // Combine
        vdst_wide += vsrc_wide;
        vres = vshrn_n_u16(vdst_wide, 8);

        vst1_lane_u32(dst, vreinterpret_u32_u8(vres), 0);
        dst++;
        src++;
        count--;
    }

    if (count) {
        uint8x8_t alpha_mask;
        static const uint8_t alpha_mask_setup[] = {3,3,3,3,7,7,7,7};
        alpha_mask = vld1_u8(alpha_mask_setup);

        do {

            uint8x8_t vsrc, vdst, vres, vsrc_alphas;
            uint16x8_t vdst_wide, vsrc_wide, vsrc_scale, vdst_scale;

            __builtin_prefetch(src+32);
            __builtin_prefetch(dst+32);

            // Load
            vsrc = vreinterpret_u8_u32(vld1_u32(src));
            vdst = vreinterpret_u8_u32(vld1_u32(dst));

            // Prepare src_scale
            vsrc_scale = vdupq_n_u16(alpha256);

            // Calc dst_scale
            vsrc_alphas = vtbl1_u8(vsrc, alpha_mask);
            vdst_scale = vmovl_u8(vsrc_alphas);
            // Calculate SkAlphaMulInv256(vdst_scale, vsrc_scale).
            // A 16-bit lane would overflow if we used 0xFFFF here,
            // so use an approximation with 0xFF00 that is off by 1,
            // and add back 1 after to get the correct value.
            // This is valid if alpha256 <= 255.
            vdst_scale = vmlsq_u16(vdupq_n_u16(0xFF00), vdst_scale, vsrc_scale);
            vdst_scale = vsraq_n_u16(vdst_scale, vdst_scale, 8);
            vdst_scale = vsraq_n_u16(vdupq_n_u16(1), vdst_scale, 8);

            // Process src
            vsrc_wide = vmovl_u8(vsrc);
            vsrc_wide *= vsrc_scale;

            // Process dst
            vdst_wide = vmovl_u8(vdst);
            vdst_wide *= vdst_scale;

            // Combine
            vdst_wide += vsrc_wide;
            vres = vshrn_n_u16(vdst_wide, 8);

            vst1_u32(dst, vreinterpret_u32_u8(vres));

            src += 2;
            dst += 2;
            count -= 2;
        } while(count);
    }
}

///////////////////////////////////////////////////////////////////////////////

#endif // #ifdef SK_CPU_ARM32

void S32A_D565_Opaque_Dither_neon (uint16_t * SK_RESTRICT dst,
                                   const SkPMColor* SK_RESTRICT src,
                                   int count, U8CPU alpha, int x, int y) {
    SkASSERT(255 == alpha);

#define    UNROLL    8

    if (count >= UNROLL) {

    uint8x8_t dbase;
    const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)];
    dbase = vld1_u8(dstart);

        do {
        uint8x8x4_t vsrc;
        uint8x8_t sr, sg, sb, sa, d;
        uint16x8_t dst8, scale8, alpha8;
        uint16x8_t dst_r, dst_g, dst_b;

#ifdef SK_CPU_ARM64
        vsrc = sk_vld4_u8_arm64_4(src);
#else
        {
        register uint8x8_t d0 asm("d0");
        register uint8x8_t d1 asm("d1");
        register uint8x8_t d2 asm("d2");
        register uint8x8_t d3 asm("d3");

        asm ("vld4.8    {d0-d3},[%[src]]! "
            : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+r" (src)
            :
        );
        vsrc.val[0] = d0;
        vsrc.val[1] = d1;
        vsrc.val[2] = d2;
        vsrc.val[3] = d3;
        }
#endif
        sa = vsrc.val[NEON_A];
        sr = vsrc.val[NEON_R];
        sg = vsrc.val[NEON_G];
        sb = vsrc.val[NEON_B];

        /* calculate 'd', which will be 0..7
         * dbase[] is 0..7; alpha is 0..256; 16 bits suffice
         */
        alpha8 = vmovl_u8(dbase);
        alpha8 = vmlal_u8(alpha8, sa, dbase);
        d = vshrn_n_u16(alpha8, 8);    // narrowing too

        // sr = sr - (sr>>5) + d
        /* watching for 8-bit overflow.  d is 0..7; risky range of
         * sr is >248; and then (sr>>5) is 7 so it offsets 'd';
         * safe  as long as we do ((sr-sr>>5) + d)
         */
        sr = vsub_u8(sr, vshr_n_u8(sr, 5));
        sr = vadd_u8(sr, d);

        // sb = sb - (sb>>5) + d
        sb = vsub_u8(sb, vshr_n_u8(sb, 5));
        sb = vadd_u8(sb, d);

        // sg = sg - (sg>>6) + d>>1; similar logic for overflows
        sg = vsub_u8(sg, vshr_n_u8(sg, 6));
        sg = vadd_u8(sg, vshr_n_u8(d,1));

        // need to pick up 8 dst's -- at 16 bits each, 128 bits
        dst8 = vld1q_u16(dst);
        dst_b = vandq_u16(dst8, vdupq_n_u16(SK_B16_MASK));
        dst_g = vshrq_n_u16(vshlq_n_u16(dst8, SK_R16_BITS), SK_R16_BITS + SK_B16_BITS);
        dst_r = vshrq_n_u16(dst8, SK_R16_SHIFT);    // clearing hi bits

        // blend
        scale8 = vsubw_u8(vdupq_n_u16(256), sa);

        // combine the addq and mul, save 3 insns
        scale8 = vshrq_n_u16(scale8, 3);
        dst_b = vmlaq_u16(vshll_n_u8(sb,2), dst_b, scale8);
        dst_g = vmlaq_u16(vshll_n_u8(sg,3), dst_g, scale8);
        dst_r = vmlaq_u16(vshll_n_u8(sr,2), dst_r, scale8);

        // repack to store
        dst8 = vshrq_n_u16(dst_b, 5);
        dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dst_g, 5), 5);
        dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dst_r,5), 11);

        vst1q_u16(dst, dst8);

        dst += UNROLL;
        count -= UNROLL;
        // skip x += UNROLL, since it's unchanged mod-4
        } while (count >= UNROLL);
    }
#undef    UNROLL

    // residuals
    if (count > 0) {
        DITHER_565_SCAN(y);
        do {
            SkPMColor c = *src++;
            SkPMColorAssert(c);
            if (c) {
                unsigned a = SkGetPackedA32(c);

                // dither and alpha are just temporary variables to work-around
                // an ICE in debug.
                unsigned dither = DITHER_VALUE(x);
                unsigned alpha = SkAlpha255To256(a);
                int d = SkAlphaMul(dither, alpha);

                unsigned sr = SkGetPackedR32(c);
                unsigned sg = SkGetPackedG32(c);
                unsigned sb = SkGetPackedB32(c);
                sr = SkDITHER_R32_FOR_565(sr, d);
                sg = SkDITHER_G32_FOR_565(sg, d);
                sb = SkDITHER_B32_FOR_565(sb, d);

                uint32_t src_expanded = (sg << 24) | (sr << 13) | (sb << 2);
                uint32_t dst_expanded = SkExpand_rgb_16(*dst);
                dst_expanded = dst_expanded * (SkAlpha255To256(255 - a) >> 3);
                // now src and dst expanded are in g:11 r:10 x:1 b:10
                *dst = SkCompact_rgb_16((src_expanded + dst_expanded) >> 5);
            }
            dst += 1;
            DITHER_INC_X(x);
        } while (--count != 0);
    }
}

///////////////////////////////////////////////////////////////////////////////

void S32_D565_Opaque_Dither_neon(uint16_t* SK_RESTRICT dst,
                                 const SkPMColor* SK_RESTRICT src,
                                 int count, U8CPU alpha, int x, int y) {
    SkASSERT(255 == alpha);

#define    UNROLL    8
    if (count >= UNROLL) {
    uint8x8_t d;
    const uint8_t *dstart = &gDitherMatrix_Neon[(y&3)*12 + (x&3)];
    d = vld1_u8(dstart);

    while (count >= UNROLL) {
        uint8x8_t sr, sg, sb;
        uint16x8_t dr, dg, db;
        uint16x8_t dst8;
        uint8x8x4_t vsrc;

#ifdef SK_CPU_ARM64
        vsrc = sk_vld4_u8_arm64_3(src);
#else
        {
        register uint8x8_t d0 asm("d0");
        register uint8x8_t d1 asm("d1");
        register uint8x8_t d2 asm("d2");
        register uint8x8_t d3 asm("d3");

        asm (
            "vld4.8    {d0-d3},[%[src]]! "
            : "=w" (d0), "=w" (d1), "=w" (d2), "=w" (d3), [src] "+&r" (src)
            :
        );
        vsrc.val[0] = d0;
        vsrc.val[1] = d1;
        vsrc.val[2] = d2;
        }
#endif
        sr = vsrc.val[NEON_R];
        sg = vsrc.val[NEON_G];
        sb = vsrc.val[NEON_B];

        /* XXX: if we want to prefetch, hide it in the above asm()
         * using the gcc __builtin_prefetch(), the prefetch will
         * fall to the bottom of the loop -- it won't stick up
         * at the top of the loop, just after the vld4.
         */

        // sr = sr - (sr>>5) + d
        sr = vsub_u8(sr, vshr_n_u8(sr, 5));
        dr = vaddl_u8(sr, d);

        // sb = sb - (sb>>5) + d
        sb = vsub_u8(sb, vshr_n_u8(sb, 5));
        db = vaddl_u8(sb, d);

        // sg = sg - (sg>>6) + d>>1; similar logic for overflows
        sg = vsub_u8(sg, vshr_n_u8(sg, 6));
        dg = vaddl_u8(sg, vshr_n_u8(d, 1));

        // pack high bits of each into 565 format  (rgb, b is lsb)
        dst8 = vshrq_n_u16(db, 3);
        dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dg, 2), 5);
        dst8 = vsliq_n_u16(dst8, vshrq_n_u16(dr, 3), 11);

        // store it
        vst1q_u16(dst, dst8);

        dst += UNROLL;
        // we don't need to increment src as the asm above has already done it
        count -= UNROLL;
        x += UNROLL;        // probably superfluous
    }
    }
#undef    UNROLL

    // residuals
    if (count > 0) {
        DITHER_565_SCAN(y);
        do {
            SkPMColor c = *src++;
            SkPMColorAssert(c);
            SkASSERT(SkGetPackedA32(c) == 255);

            unsigned dither = DITHER_VALUE(x);
            *dst++ = SkDitherRGB32To565(c, dither);
            DITHER_INC_X(x);
        } while (--count != 0);
    }
}

///////////////////////////////////////////////////////////////////////////////

const SkBlitRow::Proc16 sk_blitrow_platform_565_procs_arm_neon[] = {
    // no dither
    S32_D565_Opaque_neon,
    S32_D565_Blend_neon,
    S32A_D565_Opaque_neon,
#if 0
    S32A_D565_Blend_neon,
#else
    nullptr,   // https://code.google.com/p/skia/issues/detail?id=2797
#endif

    // dither
    S32_D565_Opaque_Dither_neon,
    S32_D565_Blend_Dither_neon,
    S32A_D565_Opaque_Dither_neon,
    nullptr,   // S32A_D565_Blend_Dither
};

const SkBlitRow::ColorProc16 sk_blitrow_platform_565_colorprocs_arm_neon[] = {
    Color32A_D565_neon,    // Color32_D565,
    Color32A_D565_neon,    // Color32A_D565,
    Color32A_D565_neon,    // Color32_D565_Dither,
    Color32A_D565_neon,    // Color32A_D565_Dither
};

const SkBlitRow::Proc32 sk_blitrow_platform_32_procs_arm_neon[] = {
    nullptr,   // S32_Opaque,
    S32_Blend_BlitRow32_neon,        // S32_Blend,
    nullptr,  // Ported to SkOpts
#ifdef SK_CPU_ARM32
    S32A_Blend_BlitRow32_neon        // S32A_Blend
#else
    nullptr
#endif
};