1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "rsCpuIntrinsic.h"
18 #include "rsCpuIntrinsicInlines.h"
19 
20 using namespace android;
21 using namespace android::renderscript;
22 
23 namespace android {
24 namespace renderscript {
25 
26 
27 class RsdCpuScriptIntrinsicBlur : public RsdCpuScriptIntrinsic {
28 public:
29     virtual void populateScript(Script *);
30     virtual void invokeFreeChildren();
31 
32     virtual void setGlobalVar(uint32_t slot, const void *data, size_t dataLength);
33     virtual void setGlobalObj(uint32_t slot, ObjectBase *data);
34 
35     virtual ~RsdCpuScriptIntrinsicBlur();
36     RsdCpuScriptIntrinsicBlur(RsdCpuReferenceImpl *ctx, const Script *s, const Element *e);
37 
38 protected:
39     float mFp[104];
40     uint16_t mIp[104];
41     void **mScratch;
42     size_t *mScratchSize;
43     float mRadius;
44     int mIradius;
45     ObjectBaseRef<Allocation> mAlloc;
46 
47     static void kernelU4(const RsForEachStubParamStruct *p,
48                          uint32_t xstart, uint32_t xend,
49                          uint32_t instep, uint32_t outstep);
50     static void kernelU1(const RsForEachStubParamStruct *p,
51                          uint32_t xstart, uint32_t xend,
52                          uint32_t instep, uint32_t outstep);
53     void ComputeGaussianWeights();
54 };
55 
56 }
57 }
58 
59 
ComputeGaussianWeights()60 void RsdCpuScriptIntrinsicBlur::ComputeGaussianWeights() {
61     memset(mFp, 0, sizeof(mFp));
62     memset(mIp, 0, sizeof(mIp));
63 
64     // Compute gaussian weights for the blur
65     // e is the euler's number
66     // TODO Define these constants only once
67     float e = 2.718281828459045f;
68     float pi = 3.1415926535897932f;
69     // g(x) = (1 / (sqrt(2 * pi) * sigma)) * e ^ (-x^2 / (2 * sigma^2))
70     // x is of the form [-radius .. 0 .. radius]
71     // and sigma varies with the radius.
72     // Based on some experimental radius values and sigmas,
73     // we approximately fit sigma = f(radius) as
74     // sigma = radius * 0.4  + 0.6
75     // The larger the radius gets, the more our gaussian blur
76     // will resemble a box blur since with large sigma
77     // the gaussian curve begins to lose its shape
78     float sigma = 0.4f * mRadius + 0.6f;
79 
80     // Now compute the coefficients. We will store some redundant values to save
81     // some math during the blur calculations precompute some values
82     float coeff1 = 1.0f / (sqrtf(2.0f * pi) * sigma);
83     float coeff2 = - 1.0f / (2.0f * sigma * sigma);
84 
85     float normalizeFactor = 0.0f;
86     float floatR = 0.0f;
87     int r;
88     mIradius = (float)ceil(mRadius) + 0.5f;
89     for (r = -mIradius; r <= mIradius; r ++) {
90         floatR = (float)r;
91         mFp[r + mIradius] = coeff1 * powf(e, floatR * floatR * coeff2);
92         normalizeFactor += mFp[r + mIradius];
93     }
94 
95     // Now we need to normalize the weights because all our coefficients need to add up to one
96     normalizeFactor = 1.0f / normalizeFactor;
97     for (r = -mIradius; r <= mIradius; r ++) {
98         mFp[r + mIradius] *= normalizeFactor;
99         mIp[r + mIradius] = (uint16_t)(mFp[r + mIradius] * 65536.0f + 0.5f);
100     }
101 }
102 
setGlobalObj(uint32_t slot,ObjectBase * data)103 void RsdCpuScriptIntrinsicBlur::setGlobalObj(uint32_t slot, ObjectBase *data) {
104     rsAssert(slot == 1);
105     mAlloc.set(static_cast<Allocation *>(data));
106 }
107 
setGlobalVar(uint32_t slot,const void * data,size_t dataLength)108 void RsdCpuScriptIntrinsicBlur::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
109     rsAssert(slot == 0);
110     mRadius = ((const float *)data)[0];
111     ComputeGaussianWeights();
112 }
113 
114 
115 
OneVU4(const RsForEachStubParamStruct * p,float4 * out,int32_t x,int32_t y,const uchar * ptrIn,int iStride,const float * gPtr,int iradius)116 static void OneVU4(const RsForEachStubParamStruct *p, float4 *out, int32_t x, int32_t y,
117                    const uchar *ptrIn, int iStride, const float* gPtr, int iradius) {
118 
119     const uchar *pi = ptrIn + x*4;
120 
121     float4 blurredPixel = 0;
122     for (int r = -iradius; r <= iradius; r ++) {
123         int validY = rsMax((y + r), 0);
124         validY = rsMin(validY, (int)(p->dimY - 1));
125         const uchar4 *pvy = (const uchar4 *)&pi[validY * iStride];
126         float4 pf = convert_float4(pvy[0]);
127         blurredPixel += pf * gPtr[0];
128         gPtr++;
129     }
130 
131     out[0] = blurredPixel;
132 }
133 
OneVU1(const RsForEachStubParamStruct * p,float * out,int32_t x,int32_t y,const uchar * ptrIn,int iStride,const float * gPtr,int iradius)134 static void OneVU1(const RsForEachStubParamStruct *p, float *out, int32_t x, int32_t y,
135                    const uchar *ptrIn, int iStride, const float* gPtr, int iradius) {
136 
137     const uchar *pi = ptrIn + x;
138 
139     float blurredPixel = 0;
140     for (int r = -iradius; r <= iradius; r ++) {
141         int validY = rsMax((y + r), 0);
142         validY = rsMin(validY, (int)(p->dimY - 1));
143         float pf = (float)pi[validY * iStride];
144         blurredPixel += pf * gPtr[0];
145         gPtr++;
146     }
147 
148     out[0] = blurredPixel;
149 }
150 
151 
152 extern "C" void rsdIntrinsicBlurU1_K(uchar *out, uchar const *in, size_t w, size_t h,
153                  size_t p, size_t x, size_t y, size_t count, size_t r, uint16_t const *tab);
154 extern "C" void rsdIntrinsicBlurU4_K(uchar4 *out, uchar4 const *in, size_t w, size_t h,
155                  size_t p, size_t x, size_t y, size_t count, size_t r, uint16_t const *tab);
156 
157 #if defined(ARCH_X86_HAVE_SSSE3)
158 extern "C" void rsdIntrinsicBlurVFU4_K(void *dst, const void *pin, int stride, const void *gptr, int rct, int x1, int ct);
159 extern "C" void rsdIntrinsicBlurHFU4_K(void *dst, const void *pin, const void *gptr, int rct, int x1, int ct);
160 extern "C" void rsdIntrinsicBlurHFU1_K(void *dst, const void *pin, const void *gptr, int rct, int x1, int ct);
161 #endif
162 
OneVFU4(float4 * out,const uchar * ptrIn,int iStride,const float * gPtr,int ct,int x1,int x2)163 static void OneVFU4(float4 *out,
164                     const uchar *ptrIn, int iStride, const float* gPtr, int ct,
165                     int x1, int x2) {
166     out += x1;
167 #if defined(ARCH_X86_HAVE_SSSE3)
168     if (gArchUseSIMD) {
169         int t = (x2 - x1);
170         t &= ~1;
171         if (t) {
172             rsdIntrinsicBlurVFU4_K(out, ptrIn, iStride, gPtr, ct, x1, x1 + t);
173         }
174         x1 += t;
175         out += t;
176         ptrIn += t << 2;
177     }
178 #endif
179     while(x2 > x1) {
180         const uchar *pi = ptrIn;
181         float4 blurredPixel = 0;
182         const float* gp = gPtr;
183 
184         for (int r = 0; r < ct; r++) {
185             float4 pf = convert_float4(((const uchar4 *)pi)[0]);
186             blurredPixel += pf * gp[0];
187             pi += iStride;
188             gp++;
189         }
190         out->xyzw = blurredPixel;
191         x1++;
192         out++;
193         ptrIn+=4;
194     }
195 }
196 
OneVFU1(float * out,const uchar * ptrIn,int iStride,const float * gPtr,int ct,int x1,int x2)197 static void OneVFU1(float *out,
198                     const uchar *ptrIn, int iStride, const float* gPtr, int ct, int x1, int x2) {
199 
200     int len = x2 - x1;
201     out += x1;
202 
203     while((x2 > x1) && (((uintptr_t)ptrIn) & 0x3)) {
204         const uchar *pi = ptrIn;
205         float blurredPixel = 0;
206         const float* gp = gPtr;
207 
208         for (int r = 0; r < ct; r++) {
209             float pf = (float)pi[0];
210             blurredPixel += pf * gp[0];
211             pi += iStride;
212             gp++;
213         }
214         out[0] = blurredPixel;
215         x1++;
216         out++;
217         ptrIn++;
218         len--;
219     }
220 #if defined(ARCH_X86_HAVE_SSSE3)
221     if (gArchUseSIMD && (x2 > x1)) {
222         int t = (x2 - x1) >> 2;
223         t &= ~1;
224         if (t) {
225             rsdIntrinsicBlurVFU4_K(out, ptrIn, iStride, gPtr, ct, 0, t );
226             len -= t << 2;
227             ptrIn += t << 2;
228             out += t << 2;
229         }
230     }
231 #endif
232     while(len > 0) {
233         const uchar *pi = ptrIn;
234         float blurredPixel = 0;
235         const float* gp = gPtr;
236 
237         for (int r = 0; r < ct; r++) {
238             float pf = (float)pi[0];
239             blurredPixel += pf * gp[0];
240             pi += iStride;
241             gp++;
242         }
243         out[0] = blurredPixel;
244         len--;
245         out++;
246         ptrIn++;
247     }
248 }
249 
OneHU4(const RsForEachStubParamStruct * p,uchar4 * out,int32_t x,const float4 * ptrIn,const float * gPtr,int iradius)250 static void OneHU4(const RsForEachStubParamStruct *p, uchar4 *out, int32_t x,
251                    const float4 *ptrIn, const float* gPtr, int iradius) {
252 
253     float4 blurredPixel = 0;
254     for (int r = -iradius; r <= iradius; r ++) {
255         int validX = rsMax((x + r), 0);
256         validX = rsMin(validX, (int)(p->dimX - 1));
257         float4 pf = ptrIn[validX];
258         blurredPixel += pf * gPtr[0];
259         gPtr++;
260     }
261 
262     out->xyzw = convert_uchar4(blurredPixel);
263 }
264 
OneHU1(const RsForEachStubParamStruct * p,uchar * out,int32_t x,const float * ptrIn,const float * gPtr,int iradius)265 static void OneHU1(const RsForEachStubParamStruct *p, uchar *out, int32_t x,
266                    const float *ptrIn, const float* gPtr, int iradius) {
267 
268     float blurredPixel = 0;
269     for (int r = -iradius; r <= iradius; r ++) {
270         int validX = rsMax((x + r), 0);
271         validX = rsMin(validX, (int)(p->dimX - 1));
272         float pf = ptrIn[validX];
273         blurredPixel += pf * gPtr[0];
274         gPtr++;
275     }
276 
277     out[0] = (uchar)blurredPixel;
278 }
279 
280 
kernelU4(const RsForEachStubParamStruct * p,uint32_t xstart,uint32_t xend,uint32_t instep,uint32_t outstep)281 void RsdCpuScriptIntrinsicBlur::kernelU4(const RsForEachStubParamStruct *p,
282                                          uint32_t xstart, uint32_t xend,
283                                          uint32_t instep, uint32_t outstep) {
284 
285     float4 stackbuf[2048];
286     float4 *buf = &stackbuf[0];
287     RsdCpuScriptIntrinsicBlur *cp = (RsdCpuScriptIntrinsicBlur *)p->usr;
288     if (!cp->mAlloc.get()) {
289         ALOGE("Blur executed without input, skipping");
290         return;
291     }
292     const uchar *pin = (const uchar *)cp->mAlloc->mHal.drvState.lod[0].mallocPtr;
293     const size_t stride = cp->mAlloc->mHal.drvState.lod[0].stride;
294 
295     uchar4 *out = (uchar4 *)p->out;
296     uint32_t x1 = xstart;
297     uint32_t x2 = xend;
298 
299 #if defined(ARCH_ARM_USE_INTRINSICS)
300     if (gArchUseSIMD) {
301         rsdIntrinsicBlurU4_K(out, (uchar4 const *)(pin + stride * p->y), p->dimX, p->dimY,
302                  stride, x1, p->y, x2 - x1, cp->mIradius, cp->mIp + cp->mIradius);
303         return;
304     }
305 #endif
306 
307     if (p->dimX > 2048) {
308         if ((p->dimX > cp->mScratchSize[p->lid]) || !cp->mScratch[p->lid]) {
309             // Pad the side of the allocation by one unit to allow alignment later
310             cp->mScratch[p->lid] = realloc(cp->mScratch[p->lid], (p->dimX + 1) * 16);
311             cp->mScratchSize[p->lid] = p->dimX;
312         }
313         // realloc only aligns to 8 bytes so we manually align to 16.
314         buf = (float4 *) ((((intptr_t)cp->mScratch[p->lid]) + 15) & ~0xf);
315     }
316     float4 *fout = (float4 *)buf;
317     int y = p->y;
318     if ((y > cp->mIradius) && (y < ((int)p->dimY - cp->mIradius))) {
319         const uchar *pi = pin + (y - cp->mIradius) * stride;
320         OneVFU4(fout, pi, stride, cp->mFp, cp->mIradius * 2 + 1, 0, p->dimX);
321     } else {
322         x1 = 0;
323         while(p->dimX > x1) {
324             OneVU4(p, fout, x1, y, pin, stride, cp->mFp, cp->mIradius);
325             fout++;
326             x1++;
327         }
328     }
329 
330     x1 = xstart;
331     while ((x1 < (uint32_t)cp->mIradius) && (x1 < x2)) {
332         OneHU4(p, out, x1, buf, cp->mFp, cp->mIradius);
333         out++;
334         x1++;
335     }
336 #if defined(ARCH_X86_HAVE_SSSE3)
337     if (gArchUseSIMD) {
338         if ((x1 + cp->mIradius) < x2) {
339             rsdIntrinsicBlurHFU4_K(out, buf - cp->mIradius, cp->mFp,
340                                    cp->mIradius * 2 + 1, x1, x2 - cp->mIradius);
341             out += (x2 - cp->mIradius) - x1;
342             x1 = x2 - cp->mIradius;
343         }
344     }
345 #endif
346     while(x2 > x1) {
347         OneHU4(p, out, x1, buf, cp->mFp, cp->mIradius);
348         out++;
349         x1++;
350     }
351 }
352 
kernelU1(const RsForEachStubParamStruct * p,uint32_t xstart,uint32_t xend,uint32_t instep,uint32_t outstep)353 void RsdCpuScriptIntrinsicBlur::kernelU1(const RsForEachStubParamStruct *p,
354                                          uint32_t xstart, uint32_t xend,
355                                          uint32_t instep, uint32_t outstep) {
356     float buf[4 * 2048];
357     RsdCpuScriptIntrinsicBlur *cp = (RsdCpuScriptIntrinsicBlur *)p->usr;
358     if (!cp->mAlloc.get()) {
359         ALOGE("Blur executed without input, skipping");
360         return;
361     }
362     const uchar *pin = (const uchar *)cp->mAlloc->mHal.drvState.lod[0].mallocPtr;
363     const size_t stride = cp->mAlloc->mHal.drvState.lod[0].stride;
364 
365     uchar *out = (uchar *)p->out;
366     uint32_t x1 = xstart;
367     uint32_t x2 = xend;
368 
369 #if defined(ARCH_ARM_USE_INTRINSICS)
370     if (gArchUseSIMD) {
371         rsdIntrinsicBlurU1_K(out, pin + stride * p->y, p->dimX, p->dimY,
372                  stride, x1, p->y, x2 - x1, cp->mIradius, cp->mIp + cp->mIradius);
373         return;
374     }
375 #endif
376 
377     float *fout = (float *)buf;
378     int y = p->y;
379     if ((y > cp->mIradius) && (y < ((int)p->dimY - cp->mIradius -1))) {
380         const uchar *pi = pin + (y - cp->mIradius) * stride;
381         OneVFU1(fout, pi, stride, cp->mFp, cp->mIradius * 2 + 1, 0, p->dimX);
382     } else {
383         x1 = 0;
384         while(p->dimX > x1) {
385             OneVU1(p, fout, x1, y, pin, stride, cp->mFp, cp->mIradius);
386             fout++;
387             x1++;
388         }
389     }
390 
391     x1 = xstart;
392     while ((x1 < x2) &&
393            ((x1 < (uint32_t)cp->mIradius) || (((uintptr_t)out) & 0x3))) {
394         OneHU1(p, out, x1, buf, cp->mFp, cp->mIradius);
395         out++;
396         x1++;
397     }
398 #if defined(ARCH_X86_HAVE_SSSE3)
399     if (gArchUseSIMD) {
400         if ((x1 + cp->mIradius) < x2) {
401             uint32_t len = x2 - (x1 + cp->mIradius);
402             len &= ~3;
403             if (len > 0) {
404                 rsdIntrinsicBlurHFU1_K(out, ((float *)buf) - cp->mIradius, cp->mFp,
405                                        cp->mIradius * 2 + 1, x1, x1 + len);
406                 out += len;
407                 x1 += len;
408             }
409         }
410     }
411 #endif
412     while(x2 > x1) {
413         OneHU1(p, out, x1, buf, cp->mFp, cp->mIradius);
414         out++;
415         x1++;
416     }
417 }
418 
RsdCpuScriptIntrinsicBlur(RsdCpuReferenceImpl * ctx,const Script * s,const Element * e)419 RsdCpuScriptIntrinsicBlur::RsdCpuScriptIntrinsicBlur(RsdCpuReferenceImpl *ctx,
420                                                      const Script *s, const Element *e)
421             : RsdCpuScriptIntrinsic(ctx, s, e, RS_SCRIPT_INTRINSIC_ID_BLUR) {
422 
423     mRootPtr = NULL;
424     if (e->getType() == RS_TYPE_UNSIGNED_8) {
425         switch (e->getVectorSize()) {
426         case 1:
427             mRootPtr = &kernelU1;
428             break;
429         case 4:
430             mRootPtr = &kernelU4;
431             break;
432         }
433     }
434     rsAssert(mRootPtr);
435     mRadius = 5;
436 
437     mScratch = new void *[mCtx->getThreadCount()];
438     mScratchSize = new size_t[mCtx->getThreadCount()];
439     memset(mScratch, 0, sizeof(void *) * mCtx->getThreadCount());
440     memset(mScratchSize, 0, sizeof(size_t) * mCtx->getThreadCount());
441 
442     ComputeGaussianWeights();
443 }
444 
~RsdCpuScriptIntrinsicBlur()445 RsdCpuScriptIntrinsicBlur::~RsdCpuScriptIntrinsicBlur() {
446     uint32_t threads = mCtx->getThreadCount();
447     if (mScratch) {
448         for (size_t i = 0; i < threads; i++) {
449             if (mScratch[i]) {
450                 free(mScratch[i]);
451             }
452         }
453         delete []mScratch;
454     }
455     if (mScratchSize) {
456         delete []mScratchSize;
457     }
458 }
459 
populateScript(Script * s)460 void RsdCpuScriptIntrinsicBlur::populateScript(Script *s) {
461     s->mHal.info.exportedVariableCount = 2;
462 }
463 
invokeFreeChildren()464 void RsdCpuScriptIntrinsicBlur::invokeFreeChildren() {
465     mAlloc.clear();
466 }
467 
468 
rsdIntrinsic_Blur(RsdCpuReferenceImpl * ctx,const Script * s,const Element * e)469 RsdCpuScriptImpl * rsdIntrinsic_Blur(RsdCpuReferenceImpl *ctx, const Script *s, const Element *e) {
470 
471     return new RsdCpuScriptIntrinsicBlur(ctx, s, e);
472 }
473 
474 
475