1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuIntrinsic.h"
18#include "rsCpuIntrinsicInlines.h"
19
20namespace android {
21namespace renderscript {
22
23class RsdCpuScriptIntrinsicBlur : public RsdCpuScriptIntrinsic {
24public:
25    void populateScript(Script *) override;
26    void invokeFreeChildren() override;
27
28    void setGlobalVar(uint32_t slot, const void *data, size_t dataLength) override;
29    void setGlobalObj(uint32_t slot, ObjectBase *data) override;
30
31    ~RsdCpuScriptIntrinsicBlur() override;
32    RsdCpuScriptIntrinsicBlur(RsdCpuReferenceImpl *ctx, const Script *s, const Element *e);
33
34protected:
35    // The size of the kernel radius is limited to 25 in ScriptIntrinsicBlur.java.
36    // So, the max kernel size is 51 (= 2 * 25 + 1).
37    // Considering SSSE3 case, which requires the size is multiple of 4,
38    // at least 52 words are necessary. Values outside of the kernel should be 0.
39    float mFp[104];
40    uint16_t mIp[104];
41    void **mScratch;
42    size_t *mScratchSize;
43    float mRadius;
44    int mIradius;
45    ObjectBaseRef<Allocation> mAlloc;
46
47    static void kernelU4(const RsExpandKernelDriverInfo *info,
48                         uint32_t xstart, uint32_t xend,
49                         uint32_t outstep);
50    static void kernelU1(const RsExpandKernelDriverInfo *info,
51                         uint32_t xstart, uint32_t xend,
52                         uint32_t outstep);
53    void ComputeGaussianWeights();
54};
55
56
57void RsdCpuScriptIntrinsicBlur::ComputeGaussianWeights() {
58    memset(mFp, 0, sizeof(mFp));
59    memset(mIp, 0, sizeof(mIp));
60
61    // Compute gaussian weights for the blur
62    // e is the euler's number
63    // TODO Define these constants only once
64    float e = 2.718281828459045f;
65    float pi = 3.1415926535897932f;
66    // g(x) = (1 / (sqrt(2 * pi) * sigma)) * e ^ (-x^2 / (2 * sigma^2))
67    // x is of the form [-radius .. 0 .. radius]
68    // and sigma varies with the radius.
69    // Based on some experimental radius values and sigmas,
70    // we approximately fit sigma = f(radius) as
71    // sigma = radius * 0.4  + 0.6
72    // The larger the radius gets, the more our gaussian blur
73    // will resemble a box blur since with large sigma
74    // the gaussian curve begins to lose its shape
75    float sigma = 0.4f * mRadius + 0.6f;
76
77    // Now compute the coefficients. We will store some redundant values to save
78    // some math during the blur calculations precompute some values
79    float coeff1 = 1.0f / (sqrtf(2.0f * pi) * sigma);
80    float coeff2 = - 1.0f / (2.0f * sigma * sigma);
81
82    float normalizeFactor = 0.0f;
83    float floatR = 0.0f;
84    int r;
85    mIradius = (float)ceil(mRadius) + 0.5f;
86    for (r = -mIradius; r <= mIradius; r ++) {
87        floatR = (float)r;
88        mFp[r + mIradius] = coeff1 * powf(e, floatR * floatR * coeff2);
89        normalizeFactor += mFp[r + mIradius];
90    }
91
92    // Now we need to normalize the weights because all our coefficients need to add up to one
93    normalizeFactor = 1.0f / normalizeFactor;
94    for (r = -mIradius; r <= mIradius; r ++) {
95        mFp[r + mIradius] *= normalizeFactor;
96        mIp[r + mIradius] = (uint16_t)(mFp[r + mIradius] * 65536.0f + 0.5f);
97    }
98}
99
100void RsdCpuScriptIntrinsicBlur::setGlobalObj(uint32_t slot, ObjectBase *data) {
101    rsAssert(slot == 1);
102    mAlloc.set(static_cast<Allocation *>(data));
103}
104
105void RsdCpuScriptIntrinsicBlur::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
106    rsAssert(slot == 0);
107    mRadius = ((const float *)data)[0];
108    ComputeGaussianWeights();
109}
110
111
112
113static void OneVU4(const RsExpandKernelDriverInfo *info, float4 *out, int32_t x, int32_t y,
114                   const uchar *ptrIn, int iStride, const float* gPtr, int iradius) {
115
116    const uchar *pi = ptrIn + x*4;
117
118    float4 blurredPixel = 0;
119    for (int r = -iradius; r <= iradius; r ++) {
120        int validY = rsMax((y + r), 0);
121        validY = rsMin(validY, (int)(info->dim.y- 1));
122        const uchar4 *pvy = (const uchar4 *)&pi[validY * iStride];
123        float4 pf = convert_float4(pvy[0]);
124        blurredPixel += pf * gPtr[0];
125        gPtr++;
126    }
127
128    out[0] = blurredPixel;
129}
130
131static void OneVU1(const RsExpandKernelDriverInfo *info, float *out, int32_t x, int32_t y,
132                   const uchar *ptrIn, int iStride, const float* gPtr, int iradius) {
133
134    const uchar *pi = ptrIn + x;
135
136    float blurredPixel = 0;
137    for (int r = -iradius; r <= iradius; r ++) {
138        int validY = rsMax((y + r), 0);
139        validY = rsMin(validY, (int)(info->dim.y - 1));
140        float pf = (float)pi[validY * iStride];
141        blurredPixel += pf * gPtr[0];
142        gPtr++;
143    }
144
145    out[0] = blurredPixel;
146}
147
148} // namespace renderscript
149} // namespace android
150
151
152extern "C" void rsdIntrinsicBlurU1_K(uchar *out, uchar const *in, size_t w, size_t h,
153                 size_t p, size_t x, size_t y, size_t count, size_t r, uint16_t const *tab);
154extern "C" void rsdIntrinsicBlurU4_K(uchar4 *out, uchar4 const *in, size_t w, size_t h,
155                 size_t p, size_t x, size_t y, size_t count, size_t r, uint16_t const *tab);
156
157#if defined(ARCH_X86_HAVE_SSSE3)
158extern void rsdIntrinsicBlurVFU4_K(void *dst, const void *pin, int stride, const void *gptr, int rct, int x1, int ct);
159extern void rsdIntrinsicBlurHFU4_K(void *dst, const void *pin, const void *gptr, int rct, int x1, int ct);
160extern void rsdIntrinsicBlurHFU1_K(void *dst, const void *pin, const void *gptr, int rct, int x1, int ct);
161#endif
162
163using android::renderscript::gArchUseSIMD;
164
165static void OneVFU4(float4 *out,
166                    const uchar *ptrIn, int iStride, const float* gPtr, int ct,
167                    int x1, int x2) {
168    out += x1;
169#if defined(ARCH_X86_HAVE_SSSE3)
170    if (gArchUseSIMD) {
171        int t = (x2 - x1);
172        t &= ~1;
173        if (t) {
174            rsdIntrinsicBlurVFU4_K(out, ptrIn, iStride, gPtr, ct, x1, x1 + t);
175        }
176        x1 += t;
177        out += t;
178        ptrIn += t << 2;
179    }
180#endif
181    while(x2 > x1) {
182        const uchar *pi = ptrIn;
183        float4 blurredPixel = 0;
184        const float* gp = gPtr;
185
186        for (int r = 0; r < ct; r++) {
187            float4 pf = convert_float4(((const uchar4 *)pi)[0]);
188            blurredPixel += pf * gp[0];
189            pi += iStride;
190            gp++;
191        }
192        out->xyzw = blurredPixel;
193        x1++;
194        out++;
195        ptrIn+=4;
196    }
197}
198
199static void OneVFU1(float *out,
200                    const uchar *ptrIn, int iStride, const float* gPtr, int ct, int x1, int x2) {
201
202    int len = x2 - x1;
203    out += x1;
204
205    while((x2 > x1) && (((uintptr_t)ptrIn) & 0x3)) {
206        const uchar *pi = ptrIn;
207        float blurredPixel = 0;
208        const float* gp = gPtr;
209
210        for (int r = 0; r < ct; r++) {
211            float pf = (float)pi[0];
212            blurredPixel += pf * gp[0];
213            pi += iStride;
214            gp++;
215        }
216        out[0] = blurredPixel;
217        x1++;
218        out++;
219        ptrIn++;
220        len--;
221    }
222#if defined(ARCH_X86_HAVE_SSSE3)
223    if (gArchUseSIMD && (x2 > x1)) {
224        int t = (x2 - x1) >> 2;
225        t &= ~1;
226        if (t) {
227            rsdIntrinsicBlurVFU4_K(out, ptrIn, iStride, gPtr, ct, 0, t );
228            len -= t << 2;
229            ptrIn += t << 2;
230            out += t << 2;
231        }
232    }
233#endif
234    while(len > 0) {
235        const uchar *pi = ptrIn;
236        float blurredPixel = 0;
237        const float* gp = gPtr;
238
239        for (int r = 0; r < ct; r++) {
240            float pf = (float)pi[0];
241            blurredPixel += pf * gp[0];
242            pi += iStride;
243            gp++;
244        }
245        out[0] = blurredPixel;
246        len--;
247        out++;
248        ptrIn++;
249    }
250}
251
252using android::renderscript::rsMin;
253using android::renderscript::rsMax;
254
255static void OneHU4(const RsExpandKernelDriverInfo *info, uchar4 *out, int32_t x,
256                   const float4 *ptrIn, const float* gPtr, int iradius) {
257
258    float4 blurredPixel = 0;
259    for (int r = -iradius; r <= iradius; r ++) {
260        int validX = rsMax((x + r), 0);
261        validX = rsMin(validX, (int)(info->dim.x - 1));
262        float4 pf = ptrIn[validX];
263        blurredPixel += pf * gPtr[0];
264        gPtr++;
265    }
266
267    out->xyzw = convert_uchar4(blurredPixel);
268}
269
270static void OneHU1(const RsExpandKernelDriverInfo *info, uchar *out, int32_t x,
271                   const float *ptrIn, const float* gPtr, int iradius) {
272
273    float blurredPixel = 0;
274    for (int r = -iradius; r <= iradius; r ++) {
275        int validX = rsMax((x + r), 0);
276        validX = rsMin(validX, (int)(info->dim.x - 1));
277        float pf = ptrIn[validX];
278        blurredPixel += pf * gPtr[0];
279        gPtr++;
280    }
281
282    out[0] = (uchar)blurredPixel;
283}
284
285
286namespace android {
287namespace renderscript {
288
289void RsdCpuScriptIntrinsicBlur::kernelU4(const RsExpandKernelDriverInfo *info,
290                                         uint32_t xstart, uint32_t xend,
291                                         uint32_t outstep) {
292
293    float4 stackbuf[2048];
294    float4 *buf = &stackbuf[0];
295    RsdCpuScriptIntrinsicBlur *cp = (RsdCpuScriptIntrinsicBlur *)info->usr;
296    if (!cp->mAlloc.get()) {
297        ALOGE("Blur executed without input, skipping");
298        return;
299    }
300    const uchar *pin = (const uchar *)cp->mAlloc->mHal.drvState.lod[0].mallocPtr;
301    const size_t stride = cp->mAlloc->mHal.drvState.lod[0].stride;
302
303    uchar4 *out = (uchar4 *)info->outPtr[0];
304    uint32_t x1 = xstart;
305    uint32_t x2 = xend;
306
307#if defined(ARCH_ARM_USE_INTRINSICS)
308    if (gArchUseSIMD && info->dim.x >= 4) {
309      rsdIntrinsicBlurU4_K(out, (uchar4 const *)(pin + stride * info->current.y),
310                 info->dim.x, info->dim.y,
311                 stride, x1, info->current.y, x2 - x1, cp->mIradius, cp->mIp + cp->mIradius);
312        return;
313    }
314#endif
315
316    if (info->dim.x > 2048) {
317        if ((info->dim.x > cp->mScratchSize[info->lid]) || !cp->mScratch[info->lid]) {
318            // Pad the side of the allocation by one unit to allow alignment later
319            cp->mScratch[info->lid] = realloc(cp->mScratch[info->lid], (info->dim.x + 1) * 16);
320            cp->mScratchSize[info->lid] = info->dim.x;
321        }
322        // realloc only aligns to 8 bytes so we manually align to 16.
323        buf = (float4 *) ((((intptr_t)cp->mScratch[info->lid]) + 15) & ~0xf);
324    }
325    float4 *fout = (float4 *)buf;
326    int y = info->current.y;
327    if ((y > cp->mIradius) && (y < ((int)info->dim.y - cp->mIradius))) {
328        const uchar *pi = pin + (y - cp->mIradius) * stride;
329        OneVFU4(fout, pi, stride, cp->mFp, cp->mIradius * 2 + 1, 0, info->dim.x);
330    } else {
331        x1 = 0;
332        while(info->dim.x > x1) {
333            OneVU4(info, fout, x1, y, pin, stride, cp->mFp, cp->mIradius);
334            fout++;
335            x1++;
336        }
337    }
338
339    x1 = xstart;
340    while ((x1 < (uint32_t)cp->mIradius) && (x1 < x2)) {
341        OneHU4(info, out, x1, buf, cp->mFp, cp->mIradius);
342        out++;
343        x1++;
344    }
345#if defined(ARCH_X86_HAVE_SSSE3)
346    if (gArchUseSIMD) {
347        if ((x1 + cp->mIradius) < x2) {
348            rsdIntrinsicBlurHFU4_K(out, buf - cp->mIradius, cp->mFp,
349                                   cp->mIradius * 2 + 1, x1, x2 - cp->mIradius);
350            out += (x2 - cp->mIradius) - x1;
351            x1 = x2 - cp->mIradius;
352        }
353    }
354#endif
355    while(x2 > x1) {
356        OneHU4(info, out, x1, buf, cp->mFp, cp->mIradius);
357        out++;
358        x1++;
359    }
360}
361
362void RsdCpuScriptIntrinsicBlur::kernelU1(const RsExpandKernelDriverInfo *info,
363                                         uint32_t xstart, uint32_t xend,
364                                         uint32_t outstep) {
365    float buf[4 * 2048];
366    RsdCpuScriptIntrinsicBlur *cp = (RsdCpuScriptIntrinsicBlur *)info->usr;
367    if (!cp->mAlloc.get()) {
368        ALOGE("Blur executed without input, skipping");
369        return;
370    }
371    const uchar *pin = (const uchar *)cp->mAlloc->mHal.drvState.lod[0].mallocPtr;
372    const size_t stride = cp->mAlloc->mHal.drvState.lod[0].stride;
373
374    uchar *out = (uchar *)info->outPtr[0];
375    uint32_t x1 = xstart;
376    uint32_t x2 = xend;
377
378#if defined(ARCH_ARM_USE_INTRINSICS)
379    if (gArchUseSIMD && info->dim.x >= 16) {
380        // The specialisation for r<=8 has an awkward prefill case, which is
381        // fiddly to resolve, where starting close to the right edge can cause
382        // a read beyond the end of input.  So avoid that case here.
383        if (cp->mIradius > 8 || (info->dim.x - rsMax(0, (int32_t)x1 - 8)) >= 16) {
384            rsdIntrinsicBlurU1_K(out, pin + stride * info->current.y, info->dim.x, info->dim.y,
385                     stride, x1, info->current.y, x2 - x1, cp->mIradius, cp->mIp + cp->mIradius);
386            return;
387        }
388    }
389#endif
390
391    float *fout = (float *)buf;
392    int y = info->current.y;
393    if ((y > cp->mIradius) && (y < ((int)info->dim.y - cp->mIradius -1))) {
394        const uchar *pi = pin + (y - cp->mIradius) * stride;
395        OneVFU1(fout, pi, stride, cp->mFp, cp->mIradius * 2 + 1, 0, info->dim.x);
396    } else {
397        x1 = 0;
398        while(info->dim.x > x1) {
399            OneVU1(info, fout, x1, y, pin, stride, cp->mFp, cp->mIradius);
400            fout++;
401            x1++;
402        }
403    }
404
405    x1 = xstart;
406    while ((x1 < x2) &&
407           ((x1 < (uint32_t)cp->mIradius) || (((uintptr_t)out) & 0x3))) {
408        OneHU1(info, out, x1, buf, cp->mFp, cp->mIradius);
409        out++;
410        x1++;
411    }
412#if defined(ARCH_X86_HAVE_SSSE3)
413    if (gArchUseSIMD) {
414        if ((x1 + cp->mIradius) < x2) {
415            uint32_t len = x2 - (x1 + cp->mIradius);
416            len &= ~3;
417
418            // rsdIntrinsicBlurHFU1_K() processes each four float values in |buf| at once, so it
419            // nees to ensure four more values can be accessed in order to avoid accessing
420            // uninitialized buffer.
421            if (len > 4) {
422                len -= 4;
423                rsdIntrinsicBlurHFU1_K(out, ((float *)buf) - cp->mIradius, cp->mFp,
424                                       cp->mIradius * 2 + 1, x1, x1 + len);
425                out += len;
426                x1 += len;
427            }
428        }
429    }
430#endif
431    while(x2 > x1) {
432        OneHU1(info, out, x1, buf, cp->mFp, cp->mIradius);
433        out++;
434        x1++;
435    }
436}
437
438RsdCpuScriptIntrinsicBlur::RsdCpuScriptIntrinsicBlur(RsdCpuReferenceImpl *ctx,
439                                                     const Script *s, const Element *e)
440            : RsdCpuScriptIntrinsic(ctx, s, e, RS_SCRIPT_INTRINSIC_ID_BLUR) {
441
442    mRootPtr = nullptr;
443    if (e->getType() == RS_TYPE_UNSIGNED_8) {
444        switch (e->getVectorSize()) {
445        case 1:
446            mRootPtr = &kernelU1;
447            break;
448        case 4:
449            mRootPtr = &kernelU4;
450            break;
451        }
452    }
453    rsAssert(mRootPtr);
454    mRadius = 5;
455
456    mScratch = new void *[mCtx->getThreadCount()];
457    mScratchSize = new size_t[mCtx->getThreadCount()];
458    memset(mScratch, 0, sizeof(void *) * mCtx->getThreadCount());
459    memset(mScratchSize, 0, sizeof(size_t) * mCtx->getThreadCount());
460
461    ComputeGaussianWeights();
462}
463
464RsdCpuScriptIntrinsicBlur::~RsdCpuScriptIntrinsicBlur() {
465    uint32_t threads = mCtx->getThreadCount();
466    if (mScratch) {
467        for (size_t i = 0; i < threads; i++) {
468            if (mScratch[i]) {
469                free(mScratch[i]);
470            }
471        }
472        delete []mScratch;
473    }
474    if (mScratchSize) {
475        delete []mScratchSize;
476    }
477}
478
479void RsdCpuScriptIntrinsicBlur::populateScript(Script *s) {
480    s->mHal.info.exportedVariableCount = 2;
481}
482
483void RsdCpuScriptIntrinsicBlur::invokeFreeChildren() {
484    mAlloc.clear();
485}
486
487RsdCpuScriptImpl * rsdIntrinsic_Blur(RsdCpuReferenceImpl *ctx, const Script *s, const Element *e) {
488
489    return new RsdCpuScriptIntrinsicBlur(ctx, s, e);
490}
491
492} // namespace renderscript
493} // namespace android
494