xref: /frameworks/rs/cpu_ref/rsCpuIntrinsics_advsimd_Resize.S

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f:
#define END(f) .size f, .-f;

/* Fixed-point precision after vertical pass -- 16 bit data minus 1 sign and 1
 * integer (bicubic has a little overshoot).  It would also be possible to add
 * a temporary DC bias to eliminate the sign bit for more precision, but that's
 * extra arithmetic.
 */
.set VERTBITS, 14

/* The size of the scratch buffer in which we store our vertically convolved
 * intermediates.
 */
.set CHUNKSHIFT, 7       /* 5 tests better for uchar4, but 7 is necessary for ridiculous (10:1) scale factors */
.set CHUNKSIZE, (1 << CHUNKSHIFT)

/* The number of components processed in a single iteration of the innermost
 * loop.
 */
.set VECSHIFT, 3
.set VECSIZE, (1<<VECSHIFT)

/* Read four different lines (except at edges where addresses may be clamped,
 * which is why we don't simply take base and stride registers), and multiply
 * and accumulate them by the coefficients in v3[0..3], leaving the results in
 * v12.  This gives eight 16-bit results representing a horizontal line of 2-8
 * input pixels (depending on number of components per pixel) to be fed into
 * the horizontal scaling pass.
 *
 * Input coefficients are 16-bit unsigned fixed-point (although [0] and [3] are
 * known to represent negative values and VMLS is used to implement this).
 * Output is VERTBITS signed fixed-point, which must leave room for a little
 * v12.  This gives eight 16-bit results.
 */
.macro vert8, dstlo=v12.4h, dsthi=v12.8h
        ld1         {v8.8b}, [x4], #8
        ld1         {v9.8b}, [x5], #8
        ld1         {v10.8b}, [x6], #8
        ld1         {v11.8b}, [x7], #8
        uxtl        v8.8h, v8.8b
        uxtl        v9.8h, v9.8b
        uxtl        v10.8h, v10.8b
        uxtl        v11.8h, v11.8b
        umull       v12.4s, v9.4h, v3.h[1]
        umull2      v13.4s, v9.8h, v3.h[1]
        umlsl       v12.4s, v8.4h, v3.h[0]
        umlsl2      v13.4s, v8.8h, v3.h[0]
        umlal       v12.4s, v10.4h, v3.h[2]
        umlal2      v13.4s, v10.8h, v3.h[2]
        umlsl       v12.4s, v11.4h, v3.h[3]
        umlsl2      v13.4s, v11.8h, v3.h[3]

        /* Shift by 8 (bits per pixel), plus 16 (the fixed-point multiplies),
         * minus VERTBITS (the number of fraction bits we want to keep from
         * here on).
         */
        sqshrn      \dstlo, v12.4s, #8 + (16 - VERTBITS)
        sqshrn2     \dsthi, v13.4s, #8 + (16 - VERTBITS)
.endm

/* As above, but only four 16-bit results into v12hi.
 */
.macro vert4, dst=v12.8h
        ld1         {v8.s}[0], [x4], #4
        ld1         {v9.s}[0], [x5], #4
        ld1         {v10.s}[0], [x6], #4
        ld1         {v11.s}[0], [x7], #4
        uxtl        v8.8h, v8.8b
        uxtl        v9.8h, v9.8b
        uxtl        v10.8h, v10.8b
        uxtl        v11.8h, v11.8b
        umull       v12.4s, v9.4h, v3.h[1]
        umlsl       v12.4s, v8.4h, v3.h[0]
        umlal       v12.4s, v10.4h, v3.h[2]
        umlsl       v12.4s, v11.4h, v3.h[3]
.ifc \dst,v12.8h
        sqshrn2     \dst, v12.4s, #8 + (16 - VERTBITS)
.else
        sqshrn      \dst, v12.4s, #8 + (16 - VERTBITS)
.endif
.endm


/* During horizontal resize having CHUNKSIZE input available means being able
 * to produce a varying amount of output, depending on the phase of the data.
 * This function calculates the minimum number of VECSIZE chunks extracted from
 * a CHUNKSIZE window (x1), and the threshold value for when the count will be
 * one higher than that (x0).
 * These work out, conveniently, to be the quotient and remainder from:
 *      (CHUNKSIZE + xinc * VECSIZE - 1) / (xinc * VECSIZE)
 *
 * The two values are packed together in a uint64_t for convenience; and
 * they are, in fact, used this way as an arithmetic short-cut later on.
 */
/* uint64_t rsdIntrinsicResize_oscctl_K(uint32_t xinc) */
ENTRY(rsdIntrinsicResize_oscctl_K)
        lsl         x2, x0, #VECSHIFT
        mov         x0, #(CHUNKSIZE << 16) - 1
        add         x0, x0, x2
        udiv        x1, x0, x2
        msub        x0, x1, x2, x0
        add         x0, x0, x1, LSL #32
        ret
END(rsdIntrinsicResize_oscctl_K)

/* Iterate to generate the uchar1, uchar2, and uchar4 versions of the code.
 * For the most part the vertical pass (the outer loop) is the same for all
 * versions.  Exceptions are handled in-line with conditional assembly.
 */
.irep comp, 1, 2, 4
.if \comp == 1
.set COMPONENT_SHIFT, 0
.elseif \comp == 2
.set COMPONENT_SHIFT, 1
.elseif \comp == 4
.set COMPONENT_SHIFT, 2
.else
.error "Unknown component count"
.endif
.set COMPONENT_COUNT, (1 << COMPONENT_SHIFT)
.set LOOP_OUTPUT_SIZE, (VECSIZE * COMPONENT_COUNT)

.set BUFFER_SIZE, (CHUNKSIZE * 2 + 4) * COMPONENT_COUNT * 2

/* void rsdIntrinsicResizeB1_K(
 *             uint8_t * restrict dst,          // x0
 *             size_t count,                    // x1
 *             uint32_t xf,                     // x2
 *             uint32_t xinc,                   // x3
 *             uint8_t const * restrict srcn,   // x4
 *             uint8_t const * restrict src0,   // x5
 *             uint8_t const * restrict src1,   // x6
 *             uint8_t const * restrict src2,   // x7
 *             size_t xclip,                    // [sp,#0]  -> [sp,#64] -> x12
 *             size_t avail,                    // [sp,#8]  -> [sp,#72] -> x11
 *             uint64_t osc_ctl,                // [sp,#16] -> [sp,#80] -> x10
 *             int32 const *yr,                 // [sp,#24] -> [sp,#88] -> v4   (copied to v3   for scalar access)
 */
ENTRY(rsdIntrinsicResizeB\comp\()_K)
            sub         x8, sp, #32
            sub         sp, sp, #64
            st1         {v8.1d - v11.1d}, [sp]
            st1         {v12.1d - v15.1d}, [x8]

            /* align the working buffer on the stack to make it easy to use bit
             * twiddling for address calculations.
             */
            sub         x12, sp, #BUFFER_SIZE
            bic         x12, x12, #(1 << (CHUNKSHIFT + 1 + COMPONENT_SHIFT + 1)) - 1

            ldr         x8, [sp,#88]            // yr
            adr         x9, 8f
            ld1         {v4.4s}, [x8]
            ld1         {v5.8h}, [x9]
            sqxtun      v4.4h, v4.4s            // yr
            dup         v6.8h, w2
            dup         v7.8h, w3
            mla         v6.8h, v5.8h, v7.8h     // vxf
            shl         v7.8h, v7.8h, #VECSHIFT // vxinc

            /* Compute starting condition for oscillator used to compute ahead
             * of time how many iterations are possible before needing to
             * refill the working buffer.  This is based on the fixed-point
             * index of the last element in the vector of pixels processed in
             * each iteration, counting up until it would overflow.
             */
            sub         x8, x2, x3
            lsl         x9, x3, #VECSHIFT
            add         x8, x8, x9

            ldr         x10, [sp,#80]           // osc_ctl
            ldp         x13,x11, [sp,#64]       // xclip, avail

            mov         x18, sp
            mov         sp, x12

            /* x4-x7 contain pointers to the four lines of input to be
             * convolved.  These pointers have been clamped vertically and
             * horizontally (which is why it's not a simple row/stride pair),
             * and the xclip argument (now in x13) indicates how many pixels
             * from true the x position of the pointer is.  This value should
             * be 0, 1, or 2 only.
             *
             * Start by placing four pixels worth of input at the far end of
             * the buffer.  As many as two of these may be clipped, so four
             * pixels are fetched, and then the first pixel is duplicated and
             * the data shifted according to xclip.  The source pointers are
             * then also adjusted according to xclip so that subsequent fetches
             * match.
             */
            mov         v3.8b, v4.8b  /* make y coeffs available for vert4 and vert8 macros */
            sub         x14, x12, x13, LSL #COMPONENT_SHIFT + 1
            add         x15, x12, #(2 * CHUNKSIZE - 4) * COMPONENT_COUNT * 2
            add         x14, x14, #4 * COMPONENT_COUNT * 2
.if \comp == 1
            vert4       v12.4h
            dup         v11.4h, v12.h[0]
            st1         {v11.4h,v12.4h}, [x12]
            ld1         {v12.4h}, [x14]
            st1         {v12.4h}, [x15]
.elseif \comp == 2
            vert8
            dup         v11.4s, v12.s[0]
            st1         {v11.8h,v12.8h}, [x12]
            ld1         {v12.8h}, [x14]
            st1         {v12.8h}, [x15]
.elseif \comp == 4
            vert8       v14.4h, v14.8h
            vert8       v15.4h, v15.8h
            dup         v12.2d, v14.d[0]
            dup         v13.2d, v14.d[0]
            st1         {v12.8h,v13.8h}, [x12], #32
            st1         {v14.8h,v15.8h}, [x12]
            sub         x12, x12, #32
            ld1         {v11.8h,v12.8h}, [x14]
            st1         {v11.8h,v12.8h}, [x15]
.endif
            /* Count off four pixels into the working buffer.
             */
            sub         x11, x11, #4
            /* Incoming pointers were to the first _legal_ pixel.  Four pixels
             * were read unconditionally, but some may have been discarded by
             * xclip, so we rewind the pointers to compensate.
             */
            sub         x4, x4, x13, LSL #COMPONENT_SHIFT
            sub         x5, x5, x13, LSL #COMPONENT_SHIFT
            sub         x6, x6, x13, LSL #COMPONENT_SHIFT
            sub         x7, x7, x13, LSL #COMPONENT_SHIFT

            /* First tap starts where we just pre-filled, at the end of the
             * buffer.
             */
            add         x2, x2, #(CHUNKSIZE * 2 - 4) << 16

            /* Use overflowing arithmetic to implement wraparound array
             * indexing.
             */
            lsl         x2, x2, #(47 - CHUNKSHIFT)
            lsl         x3, x3, #(47 - CHUNKSHIFT)


            /* Start of outermost loop.
             * Fetch CHUNKSIZE pixels into scratch buffer, then calculate the
             * number of iterations of the inner loop that can be performed and
             * get into that.
             *
             * The fill is complicated by the possibility of running out of
             * input before the scratch buffer is filled.  If this isn't a risk
             * then it's handled by the simple loop at 2:, otherwise the
             * horrible loop at 3:.
             */
1:          mov         v3.8b, v4.8b            /* put y scaling coefficients somewhere handy */
            subs        x11, x11, #CHUNKSIZE
            bge         2f                      /* if at least CHUNKSIZE are available... */
            add         x11, x11, #CHUNKSIZE    /* if they're not... */
            b           4f
            /* ..just sneaking a literal in here after this unconditional branch.. */
8:          .hword      0, 1, 2, 3, 4, 5, 6, 7
            /* basic fill loop, processing 8 bytes at a time until there are
             * fewer than eight bytes available.
             */
3:          vert8
            sub         x11, x11, #8 / COMPONENT_COUNT
            st1         {v12.8h}, [x12], #16
4:          cmp         x11, #8 / COMPONENT_COUNT - 1
            bgt         3b
.if \comp == 4
            blt         3f
            /* The last pixel (four bytes) if necessary */
            vert4
.else
            cmp         x11, #1
            blt         3f
            /* The last pixels if necessary */
            sub         x4, x4, #8
            sub         x5, x5, #8
            sub         x6, x6, #8
            sub         x7, x7, #8
            add         x4, x4, x11, LSL #COMPONENT_SHIFT
            add         x5, x5, x11, LSL #COMPONENT_SHIFT
            add         x6, x6, x11, LSL #COMPONENT_SHIFT
            add         x7, x7, x11, LSL #COMPONENT_SHIFT
            vert8
            sub         x11, sp, x11, LSL #COMPONENT_SHIFT + 1
            sub         sp, sp, #32
            sub         x11, x11, #16
.if \comp == 1
            dup         v13.8h, v12.h[7]
.elseif \comp == 2
            dup         v13.4s, v12.s[3]
.endif
            st1         {v12.8h,v13.8h}, [sp]
            ld1         {v12.8h}, [x11]
            add         sp, sp, #32
            b           4f
.endif
            /* Keep filling until we get to the end of this chunk of the buffer */
3:
.if \comp == 1
            dup         v12.8h, v12.h[7]
.elseif \comp == 2
            dup         v12.4s, v12.s[3]
.elseif \comp == 4
            dup         v12.2d, v12.d[1]
.endif
4:          st1         {v12.8h}, [x12], #16
            tst         x12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
            bne         3b
            b           4f

.align 4
2:          /* Quickly pull a chunk of data into the working buffer.
             */
            vert8
            st1         {v12.8h}, [x12], #16
            vert8
            st1         {v12.8h}, [x12], #16
            tst         x12, #(CHUNKSIZE - 1) * COMPONENT_COUNT * 2
            bne         2b
            cmp         x11, #0
            bne         3f
4:          /* if we end with 0 pixels left we'll have nothing handy to spread
             * across to the right, so we rewind a bit.
             */
            mov         x11, #1
            sub         x4, x4, #COMPONENT_COUNT
            sub         x5, x5, #COMPONENT_COUNT
            sub         x6, x6, #COMPONENT_COUNT
            sub         x7, x7, #COMPONENT_COUNT
3:          /* copy four taps (width of cubic window) to far end for overflow
             * address handling
             */
            sub         x13, x12, #CHUNKSIZE * COMPONENT_COUNT * 2
            eor         x12, x13, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
            ld1         {v14.4h}, [x13]
.elseif \comp == 2
            ld1         {v14.8h}, [x13]
.elseif \comp == 4
            ld1         {v14.8h,v15.8h}, [x13]
.endif
            add         x13, x12, #CHUNKSIZE * COMPONENT_COUNT * 2
.if \comp == 1
            st1         {v14.4h}, [x13]
.elseif \comp == 2
            st1         {v14.8h}, [x13]
.elseif \comp == 4
            st1         {v14.8h,v15.8h}, [x13]
.endif
            /* The high 32-bits of x10 contains the maximum possible iteration
             * count, but if x8 is greater than the low 32-bits of x10 then
             * this indicates that the count must be reduced by one for this
             * iteration to avoid reading past the end of the available data.
             */
            sub         x13, x10, x8
            lsr         x13, x13, #32

            madd        x8, x13, x9, x8
            sub         x8, x8, #(CHUNKSIZE << 16)

            /* prefer to count pixels, rather than vectors, to clarify the tail
             * store case on exit.
             */
            lsl         x13, x13, #VECSHIFT
            cmp         x13, x1
            csel        x13, x1, x13, gt

            sub         x1, x1, x13

            lsl         x13, x13, #COMPONENT_SHIFT

            mov         w14, #0x8000
            movi        v30.8h, #3
            dup         v31.8h, w14

            cmp         x13, #0
            bgt         3f
            cmp         x1, #0
            bgt         1b     /* an extreme case where we shouldn't use code in this structure */
            b           9f

            .align 4
2:          /* Inner loop continues here, but starts at 3:, see end of loop
             * below for explanation. */
.if LOOP_OUTPUT_SIZE == 4
            st1         {v8.s}[0], [x0], #4
.elseif LOOP_OUTPUT_SIZE == 8
            st1         {v8.8b}, [x0], #8
.elseif LOOP_OUTPUT_SIZE == 16
            st1         {v8.16b}, [x0], #16
.elseif LOOP_OUTPUT_SIZE == 32
            st1         {v8.16b,v9.16b}, [x0], #32
.endif
            /* Inner loop:  here the four x coefficients for each tap are
             * calculated in vector code, and the addresses are calculated in
             * scalar code, and these calculations are interleaved.
             */
3:          ushr        v8.8h, v6.8h, #1            // sxf
            lsr         x14, x2, #(63 - CHUNKSHIFT)
            sqrdmulh    v9.8h, v8.8h, v8.8h         // sxf**2
            add         x2, x2, x3
            sqrdmulh    v10.8h, v9.8h, v8.8h        // sxf**3
            lsr         x15, x2, #(63 - CHUNKSHIFT)
            sshll       v11.4s, v9.4h, #2
            sshll2      v12.4s, v9.8h, #2
            add         x2, x2, x3
            smlsl       v11.4s, v10.4h, v30.4h
            smlsl2      v12.4s, v10.8h, v30.8h
            lsr         x16, x2, #(63 - CHUNKSHIFT)

            shadd       v0.8h, v10.8h, v8.8h
            add         x2, x2, x3
            sub         v0.8h, v9.8h, v0.8h
            lsr         x17, x2, #(63 - CHUNKSHIFT)

            saddw       v1.4s, v11.4s, v9.4h
            saddw2      v13.4s, v12.4s, v9.8h
            add         x2, x2, x3
            shrn        v1.4h, v1.4s, #1
            shrn2       v1.8h, v13.4s, #1
            add         x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
            sub         v1.8h, v1.8h, v31.8h
            add         x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)

            saddw       v2.4s, v11.4s, v8.4h
            saddw2      v13.4s, v12.4s, v8.8h
            add         x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
            shrn        v2.4h, v2.4s, #1
            shrn2       v2.8h, v13.4s, #1
            add         x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
            neg         v2.8h, v2.8h

            shsub       v3.8h, v10.8h, v9.8h

            /* increment the x fractional parts (oveflow is ignored, as the
             * scalar arithmetic shadows this addition with full precision).
             */
            add         v6.8h, v6.8h, v7.8h

            /* At this point we have four pointers in x8-x11, pointing to the
             * four taps in the scratch buffer that must be convolved together
             * to produce an output pixel (one output pixel per pointer).
             * These pointers usually overlap, but their spacing is irregular
             * so resolving the redundancy through L1 is a pragmatic solution.
             *
             * The scratch buffer is made of signed 16-bit data, holding over
             * some extra precision, and overshoot, from the vertical pass.
             *
             * We also have the 16-bit unsigned fixed-point weights for each
             * of the four taps in v0 - v3.  That's eight pixels worth of
             * coefficients when we have only four pointers, so calculations
             * for four more pixels are interleaved with the fetch and permute
             * code for each variant in the following code.
             *
             * The data arrangement is less than ideal for any pixel format,
             * but permuting loads help to mitigate most of the problems.
             *
             * Note also that the two outside taps of a bicubic are negative,
             * but these coefficients are unsigned.  The sign is hard-coded by
             * use of multiply-and-subtract operations.
             */
.if \comp == 1
            /* The uchar 1 case.
             * Issue one lanewise ld4.h to load four consecutive pixels from
             * one pointer (one pixel) into four different registers; then load
             * four consecutive s16 values from the next pointer (pixel) into
             * the next lane of those four registers, etc., so that we finish
             * with v12 - v15 representing the four taps, and each lane
             * representing a separate pixel.
             *
             * The first ld4 uses a splat to avoid any false dependency on
             * the previous state of the register.
             */
            ld4r        {v12.8h,v13.8h,v14.8h,v15.8h}, [x14]
            lsr         x14, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.h,v13.h,v14.h,v15.h}[1], [x15]
            add         x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
            lsr         x15, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.h,v13.h,v14.h,v15.h}[2], [x16]
            add         x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
            lsr         x16, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.h,v13.h,v14.h,v15.h}[3], [x17]
            add         x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
            lsr         x17, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.h,v13.h,v14.h,v15.h}[4], [x14]
            add         x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
            ld4         {v12.h,v13.h,v14.h,v15.h}[5], [x15]
            ld4         {v12.h,v13.h,v14.h,v15.h}[6], [x16]
            ld4         {v12.h,v13.h,v14.h,v15.h}[7], [x17]

            smull       v8.4s, v12.4h, v0.4h
            smull2      v9.4s, v12.8h, v0.8h
            smlsl       v8.4s, v13.4h, v1.4h
            smlsl2      v9.4s, v13.8h, v1.8h
            smlsl       v8.4s, v14.4h, v2.4h
            smlsl2      v9.4s, v14.8h, v2.8h
            smlal       v8.4s, v15.4h, v3.4h
            smlal2      v9.4s, v15.8h, v3.8h

            subs        x13, x13, #LOOP_OUTPUT_SIZE

            sqrshrn     v8.4h, v8.4s, #15
            sqrshrn2    v8.8h, v9.4s, #15

            sqrshrun    v8.8b, v8.8h, #VERTBITS - 8
.elseif \comp == 2
            /* The uchar2 case:
             * This time load pairs of values into adjacent lanes in v12 - v15
             * by aliasing them as u32 data; leaving room for only four pixels,
             * so the process has to be done twice.  This also means that the
             * coefficient registers fail to align with the coefficient data
             * (eight separate pixels), so that has to be doubled-up to match.
             */
            ld4r        {v12.4s,v13.4s,v14.4s,v15.4s}, [x14]
            lsr         x14, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.s,v13.s,v14.s,v15.s}[1], [x15]
            add         x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
            lsr         x15, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.s,v13.s,v14.s,v15.s}[2], [x16]
            add         x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
            lsr         x16, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld4         {v12.s,v13.s,v14.s,v15.s}[3], [x17]
            add         x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
            lsr         x17, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3

            /* double-up coefficients to align with component pairs */
            zip1        v16.8h, v0.8h, v0.8h
            add         x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
            zip1        v17.8h, v1.8h, v1.8h
            zip1        v18.8h, v2.8h, v2.8h
            zip1        v19.8h, v3.8h, v3.8h

            smull       v8.4s, v12.4h, v16.4h
            smull2      v9.4s, v12.8h, v16.8h
            smlsl       v8.4s, v13.4h, v17.4h
            smlsl2      v9.4s, v13.8h, v17.8h
            smlsl       v8.4s, v14.4h, v18.4h
            smlsl2      v9.4s, v14.8h, v18.8h
            smlal       v8.4s, v15.4h, v19.4h
            smlal2      v9.4s, v15.8h, v19.8h

            sqrshrn     v8.4h, v8.4s, #15
            sqrshrn2    v8.8h, v9.4s, #15

            ld4r        {v12.4s,v13.4s,v14.4s,v15.4s}, [x14]
            ld4         {v12.s,v13.s,v14.s,v15.s}[1], [x15]
            ld4         {v12.s,v13.s,v14.s,v15.s}[2], [x16]
            ld4         {v12.s,v13.s,v14.s,v15.s}[3], [x17]

            /* double-up coefficients to align with component pairs */
            zip2        v16.8h, v0.8h, v0.8h
            zip2        v17.8h, v1.8h, v1.8h
            zip2        v18.8h, v2.8h, v2.8h
            zip2        v19.8h, v3.8h, v3.8h

            smull       v10.4s, v12.4h, v16.4h
            smull2      v11.4s, v12.8h, v16.8h
            smlsl       v10.4s, v13.4h, v17.4h
            smlsl2      v11.4s, v13.8h, v17.8h
            smlsl       v10.4s, v14.4h, v18.4h
            smlsl2      v11.4s, v14.8h, v18.8h
            smlal       v10.4s, v15.4h, v19.4h
            smlal2      v11.4s, v15.8h, v19.8h

            subs        x13, x13, #LOOP_OUTPUT_SIZE

            sqrshrn     v9.4h, v10.4s, #15
            sqrshrn2    v9.8h, v11.4s, #15

            sqrshrun     v8.8b, v8.8h, #VERTBITS - 8
            sqrshrun2    v8.16b, v9.8h, #VERTBITS - 8
.elseif \comp == 4
            /* The uchar4 case.
             * This case is comparatively painless because four s16s are the
             * smallest addressable unit for a vmul-by-scalar.  Rather than
             * permute the data, simply arrange the multiplies to suit the way
             * the data comes in.  That's a lot of data, though, so things
             * progress in pairs of pixels at a time.
             */
            ld1         {v12.8h,v13.8h}, [x14]
            lsr         x14, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld1         {v14.8h,v15.8h}, [x15]
            add         x14, sp, x14, LSL #(COMPONENT_SHIFT + 1)
            lsr         x15, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3

            smull       v8.4s, v12.4h, v0.h[0]
            smull       v9.4s, v14.4h, v0.h[1]
            smlsl2      v8.4s, v12.8h, v1.h[0]
            smlsl2      v9.4s, v14.8h, v1.h[1]
            smlsl       v8.4s, v13.4h, v2.h[0]
            smlsl       v9.4s, v15.4h, v2.h[1]
            smlal2      v8.4s, v13.8h, v3.h[0]
            smlal2      v9.4s, v15.8h, v3.h[1]

            /* And two more...  */
            ld1         {v12.8h,v13.8h}, [x16]
            add         x15, sp, x15, LSL #(COMPONENT_SHIFT + 1)
            lsr         x16, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3
            ld1         {v14.8h,v15.8h}, [x17]
            add         x16, sp, x16, LSL #(COMPONENT_SHIFT + 1)
            lsr         x17, x2, #(63 - CHUNKSHIFT)
            add         x2, x2, x3

            sqrshrn     v8.4h, v8.4s, #15
            add         x17, sp, x17, LSL #(COMPONENT_SHIFT + 1)
            sqrshrn2    v8.8h, v9.4s, #15

            smull       v10.4s, v12.4h, v0.h[2]
            smull       v11.4s, v14.4h, v0.h[3]
            smlsl2      v10.4s, v12.8h, v1.h[2]
            smlsl2      v11.4s, v14.8h, v1.h[3]
            smlsl       v10.4s, v13.4h, v2.h[2]
            smlsl       v11.4s, v15.4h, v2.h[3]
            smlal2      v10.4s, v13.8h, v3.h[2]
            smlal2      v11.4s, v15.8h, v3.h[3]

            sqrshrn     v9.4h, v10.4s, #15
            sqrshrn2    v9.8h, v11.4s, #15

            sqrshrun     v8.8b, v8.8h, #VERTBITS - 8
            sqrshrun2    v8.16b, v9.8h, #VERTBITS - 8

            /* And two more...  */
            ld1         {v12.8h,v13.8h}, [x14]
            ld1         {v14.8h,v15.8h}, [x15]

            smull       v10.4s, v12.4h, v0.h[4]
            smull       v11.4s, v14.4h, v0.h[5]
            smlsl2      v10.4s, v12.8h, v1.h[4]
            smlsl2      v11.4s, v14.8h, v1.h[5]
            smlsl       v10.4s, v13.4h, v2.h[4]
            smlsl       v11.4s, v15.4h, v2.h[5]
            smlal2      v10.4s, v13.8h, v3.h[4]
            smlal2      v11.4s, v15.8h, v3.h[5]

            /* And two more...  */
            ld1         {v12.8h,v13.8h}, [x16]
            ld1         {v14.8h,v15.8h}, [x17]

            subs        x13, x13, #LOOP_OUTPUT_SIZE

            sqrshrn     v9.4h, v10.4s, #15
            sqrshrn2    v9.8h, v11.4s, #15

            smull       v10.4s, v12.4h, v0.h[6]
            smull       v11.4s, v14.4h, v0.h[7]
            smlsl2      v10.4s, v12.8h, v1.h[6]
            smlsl2      v11.4s, v14.8h, v1.h[7]
            smlsl       v10.4s, v13.4h, v2.h[6]
            smlsl       v11.4s, v15.4h, v2.h[7]
            smlal2      v10.4s, v13.8h, v3.h[6]
            smlal2      v11.4s, v15.8h, v3.h[7]

            sqrshrn     v10.4h, v10.4s, #15
            sqrshrn2    v10.8h, v11.4s, #15

            sqrshrun     v9.8b, v9.8h, #VERTBITS - 8
            sqrshrun2    v9.16b, v10.8h, #VERTBITS - 8
.endif
            bgt         2b      /* continue inner loop */
            /* The inner loop has already been limited to ensure that none of
             * the earlier iterations could overfill the output, so the store
             * appears within the loop but after the conditional branch (at the
             * top).  At the end, provided it won't overfill, perform the final
             * store here.  If it would, then break out to the tricky tail case
             * instead.
             */
            blt         1f
            /* Store the amount of data appropriate to the configuration of the
             * instance being assembled.
             */
.if LOOP_OUTPUT_SIZE == 4
            st1         {v8.s}[0], [x0], #4
.elseif LOOP_OUTPUT_SIZE == 8
            st1         {v8.8b}, [x0], #8
.elseif LOOP_OUTPUT_SIZE == 16
            st1         {v8.16b}, [x0], #16
.elseif LOOP_OUTPUT_SIZE == 32
            st1         {v8.16b,v9.16b}, [x0], #32
.endif
            b           1b              /* resume outer loop */
            /* Partial tail store case:
             * Different versions of the code need different subsets of the
             * following partial stores.  Here the number of components and the
             * size of the chunk of data produced by each inner loop iteration
             * is tested to figure out whether or not each phrase is relevant.
             */
.if 16 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 16
1:          tst         x13, #16
            beq         1f
            st1         {v8.16b}, [x0], #16
            mov         v8.16b, v9.16b
.endif
.if 8 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 8
1:          tst         x13, #8
            beq         1f
            st1         {v8.8b}, [x0], #8
            ext         v8.16b, v8.16b, v8.16b, #8
.endif
.if 4 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 4
1:          tst         x13, #4
            beq         1f
            st1         {v8.s}[0], [x0], #4
            ext         v8.8b, v8.8b, v8.8b, #4
.endif
.if 2 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 2
1:          tst         x13, #2
            beq         1f
            st1         {v8.h}[0], [x0], #2
            ext         v8.8b, v8.8b, v8.8b, #2
.endif
.if 1 < LOOP_OUTPUT_SIZE && COMPONENT_COUNT <= 1
1:          tst         x13, #1
            beq         1f
            st1         {v8.b}[0], [x0], #1
.endif
1:
9:          mov         sp, x18
            ld1         {v8.1d - v11.1d}, [sp], #32
            ld1         {v12.1d - v15.1d}, [sp], #32
            ret
END(rsdIntrinsicResizeB\comp\()_K)
.endr