1 // Copyright 2015 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // NEON version of rescaling functions
11 //
12 // Author: Skal (pascal.massimino@gmail.com)
13 
14 #include "./dsp.h"
15 
16 #if defined(WEBP_USE_NEON)
17 
18 #include <arm_neon.h>
19 #include <assert.h>
20 #include "./neon.h"
21 #include "../utils/rescaler.h"
22 
23 #define ROUNDER (WEBP_RESCALER_ONE >> 1)
24 #define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
25 
26 #define LOAD_32x4(SRC, DST) const uint32x4_t DST = vld1q_u32((SRC))
27 #define LOAD_32x8(SRC, DST0, DST1)                                    \
28     LOAD_32x4(SRC + 0, DST0);                                         \
29     LOAD_32x4(SRC + 4, DST1)
30 
31 #define STORE_32x8(SRC0, SRC1, DST) do {                              \
32     vst1q_u32((DST) + 0, SRC0);                                       \
33     vst1q_u32((DST) + 4, SRC1);                                       \
34 } while (0);
35 
36 #if (WEBP_RESCALER_RFIX == 32)
37 #define MAKE_HALF_CST(C) vdupq_n_s32((int32_t)((C) >> 1))
38 #define MULT_FIX(A, B) /* note: B is actualy scale>>1. See MAKE_HALF_CST */ \
39     vreinterpretq_u32_s32(vqrdmulhq_s32(vreinterpretq_s32_u32((A)), (B)))
40 #else
41 #error "MULT_FIX/WEBP_RESCALER_RFIX need some more work"
42 #endif
43 
Interpolate(const rescaler_t * const frow,const rescaler_t * const irow,uint32_t A,uint32_t B)44 static uint32x4_t Interpolate(const rescaler_t* const frow,
45                               const rescaler_t* const irow,
46                               uint32_t A, uint32_t B) {
47   LOAD_32x4(frow, A0);
48   LOAD_32x4(irow, B0);
49   const uint64x2_t C0 = vmull_n_u32(vget_low_u32(A0), A);
50   const uint64x2_t C1 = vmull_n_u32(vget_high_u32(A0), A);
51   const uint64x2_t D0 = vmlal_n_u32(C0, vget_low_u32(B0), B);
52   const uint64x2_t D1 = vmlal_n_u32(C1, vget_high_u32(B0), B);
53   const uint32x4_t E = vcombine_u32(
54       vrshrn_n_u64(D0, WEBP_RESCALER_RFIX),
55       vrshrn_n_u64(D1, WEBP_RESCALER_RFIX));
56   return E;
57 }
58 
RescalerExportRowExpand(WebPRescaler * const wrk)59 static void RescalerExportRowExpand(WebPRescaler* const wrk) {
60   int x_out;
61   uint8_t* const dst = wrk->dst;
62   rescaler_t* const irow = wrk->irow;
63   const int x_out_max = wrk->dst_width * wrk->num_channels;
64   const int max_span = x_out_max & ~7;
65   const rescaler_t* const frow = wrk->frow;
66   const uint32_t fy_scale = wrk->fy_scale;
67   const int32x4_t fy_scale_half = MAKE_HALF_CST(fy_scale);
68   assert(!WebPRescalerOutputDone(wrk));
69   assert(wrk->y_accum <= 0);
70   assert(wrk->y_expand);
71   assert(wrk->y_sub != 0);
72   if (wrk->y_accum == 0) {
73     for (x_out = 0; x_out < max_span; x_out += 8) {
74       LOAD_32x4(frow + x_out + 0, A0);
75       LOAD_32x4(frow + x_out + 4, A1);
76       const uint32x4_t B0 = MULT_FIX(A0, fy_scale_half);
77       const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half);
78       const uint16x4_t C0 = vmovn_u32(B0);
79       const uint16x4_t C1 = vmovn_u32(B1);
80       const uint8x8_t D = vmovn_u16(vcombine_u16(C0, C1));
81       vst1_u8(dst + x_out, D);
82     }
83     for (; x_out < x_out_max; ++x_out) {
84       const uint32_t J = frow[x_out];
85       const int v = (int)MULT_FIX_C(J, fy_scale);
86       assert(v >= 0 && v <= 255);
87       dst[x_out] = v;
88     }
89   } else {
90     const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
91     const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B);
92     for (x_out = 0; x_out < max_span; x_out += 8) {
93       const uint32x4_t C0 =
94           Interpolate(frow + x_out + 0, irow + x_out + 0, A, B);
95       const uint32x4_t C1 =
96           Interpolate(frow + x_out + 4, irow + x_out + 4, A, B);
97       const uint32x4_t D0 = MULT_FIX(C0, fy_scale_half);
98       const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half);
99       const uint16x4_t E0 = vmovn_u32(D0);
100       const uint16x4_t E1 = vmovn_u32(D1);
101       const uint8x8_t F = vmovn_u16(vcombine_u16(E0, E1));
102       vst1_u8(dst + x_out, F);
103     }
104     for (; x_out < x_out_max; ++x_out) {
105       const uint64_t I = (uint64_t)A * frow[x_out]
106                        + (uint64_t)B * irow[x_out];
107       const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
108       const int v = (int)MULT_FIX_C(J, fy_scale);
109       assert(v >= 0 && v <= 255);
110       dst[x_out] = v;
111     }
112   }
113 }
114 
RescalerExportRowShrink(WebPRescaler * const wrk)115 static void RescalerExportRowShrink(WebPRescaler* const wrk) {
116   int x_out;
117   uint8_t* const dst = wrk->dst;
118   rescaler_t* const irow = wrk->irow;
119   const int x_out_max = wrk->dst_width * wrk->num_channels;
120   const int max_span = x_out_max & ~7;
121   const rescaler_t* const frow = wrk->frow;
122   const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
123   const uint32_t fxy_scale = wrk->fxy_scale;
124   const uint32x4_t zero = vdupq_n_u32(0);
125   const int32x4_t yscale_half = MAKE_HALF_CST(yscale);
126   const int32x4_t fxy_scale_half = MAKE_HALF_CST(fxy_scale);
127   assert(!WebPRescalerOutputDone(wrk));
128   assert(wrk->y_accum <= 0);
129   assert(!wrk->y_expand);
130   if (yscale) {
131     for (x_out = 0; x_out < max_span; x_out += 8) {
132       LOAD_32x8(frow + x_out, in0, in1);
133       LOAD_32x8(irow + x_out, in2, in3);
134       const uint32x4_t A0 = MULT_FIX(in0, yscale_half);
135       const uint32x4_t A1 = MULT_FIX(in1, yscale_half);
136       const uint32x4_t B0 = vqsubq_u32(in2, A0);
137       const uint32x4_t B1 = vqsubq_u32(in3, A1);
138       const uint32x4_t C0 = MULT_FIX(B0, fxy_scale_half);
139       const uint32x4_t C1 = MULT_FIX(B1, fxy_scale_half);
140       const uint16x4_t D0 = vmovn_u32(C0);
141       const uint16x4_t D1 = vmovn_u32(C1);
142       const uint8x8_t E = vmovn_u16(vcombine_u16(D0, D1));
143       vst1_u8(dst + x_out, E);
144       STORE_32x8(A0, A1, irow + x_out);
145     }
146     for (; x_out < x_out_max; ++x_out) {
147       const uint32_t frac = (uint32_t)MULT_FIX_C(frow[x_out], yscale);
148       const int v = (int)MULT_FIX_C(irow[x_out] - frac, wrk->fxy_scale);
149       assert(v >= 0 && v <= 255);
150       dst[x_out] = v;
151       irow[x_out] = frac;   // new fractional start
152     }
153   } else {
154     for (x_out = 0; x_out < max_span; x_out += 8) {
155       LOAD_32x8(irow + x_out, in0, in1);
156       const uint32x4_t A0 = MULT_FIX(in0, fxy_scale_half);
157       const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half);
158       const uint16x4_t B0 = vmovn_u32(A0);
159       const uint16x4_t B1 = vmovn_u32(A1);
160       const uint8x8_t C = vmovn_u16(vcombine_u16(B0, B1));
161       vst1_u8(dst + x_out, C);
162       STORE_32x8(zero, zero, irow + x_out);
163     }
164     for (; x_out < x_out_max; ++x_out) {
165       const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale);
166       assert(v >= 0 && v <= 255);
167       dst[x_out] = v;
168       irow[x_out] = 0;
169     }
170   }
171 }
172 
173 //------------------------------------------------------------------------------
174 
175 extern void WebPRescalerDspInitNEON(void);
176 
WebPRescalerDspInitNEON(void)177 WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitNEON(void) {
178   WebPRescalerExportRowExpand = RescalerExportRowExpand;
179   WebPRescalerExportRowShrink = RescalerExportRowShrink;
180 }
181 
182 #else     // !WEBP_USE_NEON
183 
184 WEBP_DSP_INIT_STUB(WebPRescalerDspInitNEON)
185 
186 #endif    // WEBP_USE_NEON
187