1/*
2 * Copyright 2015 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8#include "SkOpts.h"
9
10#define SK_OPTS_NS sk_sse41
11#include "SkBlurImageFilter_opts.h"
12
13#ifndef SK_SUPPORT_LEGACY_X86_BLITS
14
15namespace sk_sse41 {
16
17// An SSE register holding at most 64 bits of useful data in the low lanes.
18struct m64i {
19    __m128i v;
20    /*implicit*/ m64i(__m128i v) : v(v) {}
21    operator __m128i() const { return v; }
22};
23
24// Load 4, 2, or 1 constant pixels or coverages (4x replicated).
25static __m128i next4(uint32_t val) { return _mm_set1_epi32(val); }
26static m64i    next2(uint32_t val) { return _mm_set1_epi32(val); }
27static m64i    next1(uint32_t val) { return _mm_set1_epi32(val); }
28
29static __m128i next4(uint8_t val) { return _mm_set1_epi8(val); }
30static m64i    next2(uint8_t val) { return _mm_set1_epi8(val); }
31static m64i    next1(uint8_t val) { return _mm_set1_epi8(val); }
32
33// Load 4, 2, or 1 variable pixels or coverages (4x replicated),
34// incrementing the pointer past what we read.
35static __m128i next4(const uint32_t*& ptr) {
36    auto r = _mm_loadu_si128((const __m128i*)ptr);
37    ptr += 4;
38    return r;
39}
40static m64i next2(const uint32_t*& ptr) {
41    auto r = _mm_loadl_epi64((const __m128i*)ptr);
42    ptr += 2;
43    return r;
44}
45static m64i next1(const uint32_t*& ptr) {
46    auto r = _mm_cvtsi32_si128(*ptr);
47    ptr += 1;
48    return r;
49}
50
51// xyzw -> xxxx yyyy zzzz wwww
52static __m128i replicate_coverage(__m128i xyzw) {
53    const uint8_t mask[] = { 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 };
54    return _mm_shuffle_epi8(xyzw, _mm_load_si128((const __m128i*)mask));
55}
56
57static __m128i next4(const uint8_t*& ptr) {
58    auto r = replicate_coverage(_mm_cvtsi32_si128(*(const uint32_t*)ptr));
59    ptr += 4;
60    return r;
61}
62static m64i next2(const uint8_t*& ptr) {
63    auto r = replicate_coverage(_mm_cvtsi32_si128(*(const uint16_t*)ptr));
64    ptr += 2;
65    return r;
66}
67static m64i next1(const uint8_t*& ptr) {
68    auto r = replicate_coverage(_mm_cvtsi32_si128(*ptr));
69    ptr += 1;
70    return r;
71}
72
73// For i = 0...n, tgt = fn(dst,src,cov), where Dst,Src,and Cov can be constants or arrays.
74template <typename Dst, typename Src, typename Cov, typename Fn>
75static void loop(int n, uint32_t* t, const Dst dst, const Src src, const Cov cov, Fn&& fn) {
76    // We don't want to muck with the callers' pointers, so we make them const and copy here.
77    Dst d = dst;
78    Src s = src;
79    Cov c = cov;
80
81    // Writing this as a single while-loop helps hoist loop invariants from fn.
82    while (n) {
83        if (n >= 4) {
84            _mm_storeu_si128((__m128i*)t, fn(next4(d), next4(s), next4(c)));
85            t += 4;
86            n -= 4;
87            continue;
88        }
89        if (n & 2) {
90            _mm_storel_epi64((__m128i*)t, fn(next2(d), next2(s), next2(c)));
91            t += 2;
92        }
93        if (n & 1) {
94            *t = _mm_cvtsi128_si32(fn(next1(d), next1(s), next1(c)));
95        }
96        return;
97    }
98}
99
100//                                             packed
101// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ //
102//                                            unpacked
103
104// Everything on the packed side of the squiggly line deals with densely packed 8-bit data,
105// e.g. [BGRA bgra ... ] for pixels or [ CCCC cccc ... ] for coverage.
106//
107// Everything on the unpacked side of the squiggly line deals with unpacked 8-bit data,
108// e.g [B_G_ R_A_ b_g_ r_a_ ] for pixels or [ C_C_ C_C_ c_c_ c_c_ c_c_ ] for coverage,
109// where _ is a zero byte.
110//
111// Adapt<Fn> / adapt(fn) allow the two sides to interoperate,
112// by unpacking arguments, calling fn, then packing the results.
113//
114// This lets us write most of our code in terms of unpacked inputs (considerably simpler)
115// and all the packing and unpacking is handled automatically.
116
117template <typename Fn>
118struct Adapt {
119    Fn fn;
120
121    __m128i operator()(__m128i d, __m128i s, __m128i c) {
122        auto lo = [](__m128i x) { return _mm_unpacklo_epi8(x, _mm_setzero_si128()); };
123        auto hi = [](__m128i x) { return _mm_unpackhi_epi8(x, _mm_setzero_si128()); };
124        return _mm_packus_epi16(fn(lo(d), lo(s), lo(c)),
125                                fn(hi(d), hi(s), hi(c)));
126    }
127
128    m64i operator()(const m64i& d, const m64i& s, const m64i& c) {
129        auto lo = [](__m128i x) { return _mm_unpacklo_epi8(x, _mm_setzero_si128()); };
130        auto r = fn(lo(d), lo(s), lo(c));
131        return _mm_packus_epi16(r, r);
132    }
133};
134
135template <typename Fn>
136static Adapt<Fn> adapt(Fn&& fn) { return { fn }; }
137
138// These helpers all work exclusively with unpacked 8-bit values,
139// except div255() with is 16-bit -> unpacked 8-bit, and mul255() which is the reverse.
140
141// Divide by 255 with rounding.
142// (x+127)/255 == ((x+128)*257)>>16.
143// Sometimes we can be more efficient by breaking this into two parts.
144static __m128i div255_part1(__m128i x) { return _mm_add_epi16(x, _mm_set1_epi16(128)); }
145static __m128i div255_part2(__m128i x) { return _mm_mulhi_epu16(x, _mm_set1_epi16(257)); }
146static __m128i div255(__m128i x) { return div255_part2(div255_part1(x)); }
147
148// (x*y+127)/255, a byte multiply.
149static __m128i scale(__m128i x, __m128i y) { return div255(_mm_mullo_epi16(x, y)); }
150
151// (255 * x).
152static __m128i mul255(__m128i x) { return _mm_sub_epi16(_mm_slli_epi16(x, 8), x); }
153
154// (255 - x).
155static __m128i inv(__m128i x) { return _mm_xor_si128(_mm_set1_epi16(0x00ff), x); }
156
157// ARGB argb -> AAAA aaaa
158static __m128i alphas(__m128i px) {
159    const int a = 2 * (SK_A32_SHIFT/8);  // SK_A32_SHIFT is typically 24, so this is typically 6.
160    const int _ = ~0;
161    return _mm_shuffle_epi8(px, _mm_setr_epi8(a+0,_,a+0,_,a+0,_,a+0,_, a+8,_,a+8,_,a+8,_,a+8,_));
162}
163
164// SrcOver, with a constant source and full coverage.
165static void blit_row_color32(SkPMColor* tgt, const SkPMColor* dst, int n, SkPMColor src) {
166    // We want to calculate s + (d * inv(alphas(s)) + 127)/255.
167    // We'd generally do that div255 as s + ((d * inv(alphas(s)) + 128)*257)>>16.
168
169    // But we can go one step further to ((s*255 + 128 + d*inv(alphas(s)))*257)>>16.
170    // This lets us hoist (s*255+128) and inv(alphas(s)) out of the loop.
171    __m128i s = _mm_unpacklo_epi8(_mm_set1_epi32(src), _mm_setzero_si128()),
172            s_255_128 = div255_part1(mul255(s)),
173            A = inv(alphas(s));
174
175    const uint8_t cov = 0xff;
176    loop(n, tgt, dst, src, cov, adapt([=](__m128i d, __m128i, __m128i) {
177        return div255_part2(_mm_add_epi16(s_255_128, _mm_mullo_epi16(d, A)));
178    }));
179}
180
181// SrcOver, with a constant source and variable coverage.
182// If the source is opaque, SrcOver becomes Src.
183static void blit_mask_d32_a8(SkPMColor* dst,     size_t dstRB,
184                             const SkAlpha* cov, size_t covRB,
185                             SkColor color, int w, int h) {
186    if (SkColorGetA(color) == 0xFF) {
187        const SkPMColor src = SkSwizzle_BGRA_to_PMColor(color);
188        while (h --> 0) {
189            loop(w, dst, (const SkPMColor*)dst, src, cov,
190                    adapt([](__m128i d, __m128i s, __m128i c) {
191                // Src blend mode: a simple lerp from d to s by c.
192                // TODO: try a pmaddubsw version?
193                return div255(_mm_add_epi16(_mm_mullo_epi16(inv(c),d),
194                                            _mm_mullo_epi16(    c ,s)));
195            }));
196            dst += dstRB / sizeof(*dst);
197            cov += covRB / sizeof(*cov);
198        }
199    } else {
200        const SkPMColor src = SkPreMultiplyColor(color);
201        while (h --> 0) {
202            loop(w, dst, (const SkPMColor*)dst, src, cov,
203                    adapt([](__m128i d, __m128i s, __m128i c) {
204                // SrcOver blend mode, with coverage folded into source alpha.
205                __m128i sc = scale(s,c),
206                        AC = inv(alphas(sc));
207                return _mm_add_epi16(sc, scale(d,AC));
208            }));
209            dst += dstRB / sizeof(*dst);
210            cov += covRB / sizeof(*cov);
211        }
212    }
213}
214
215}  // namespace sk_sse41
216
217#endif
218
219namespace SkOpts {
220    void Init_sse41() {
221        box_blur_xx = sk_sse41::box_blur_xx;
222        box_blur_xy = sk_sse41::box_blur_xy;
223        box_blur_yx = sk_sse41::box_blur_yx;
224
225    #ifndef SK_SUPPORT_LEGACY_X86_BLITS
226        blit_row_color32 = sk_sse41::blit_row_color32;
227        blit_mask_d32_a8 = sk_sse41::blit_mask_d32_a8;
228    #endif
229    }
230}
231