1/* 2Copyright (c) 2011 Intel Corporation 3All rights reserved. 4 5Redistribution and use in source and binary forms, with or without 6modification, are permitted provided that the following conditions are met: 7 8 * Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 11 * Redistributions in binary form must reproduce the above copyright notice, 12 * this list of conditions and the following disclaimer in the documentation 13 * and/or other materials provided with the distribution. 14 15 * Neither the name of Intel Corporation nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 19THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 23ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 26ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29*/ 30 31#ifndef L 32# define L(label) .L##label 33#endif 34 35#ifndef cfi_startproc 36# define cfi_startproc .cfi_startproc 37#endif 38 39#ifndef cfi_endproc 40# define cfi_endproc .cfi_endproc 41#endif 42 43#ifndef cfi_rel_offset 44# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off 45#endif 46 47#ifndef cfi_restore 48# define cfi_restore(reg) .cfi_restore reg 49#endif 50 51#ifndef cfi_adjust_cfa_offset 52# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off 53#endif 54 55#ifndef ENTRY 56# define ENTRY(name) \ 57 .type name, @function; \ 58 .globl name; \ 59 .p2align 4; \ 60name: \ 61 cfi_startproc 62#endif 63 64#ifndef END 65# define END(name) \ 66 cfi_endproc; \ 67 .size name, .-name 68#endif 69 70#define CFI_PUSH(REG) \ 71 cfi_adjust_cfa_offset (4); \ 72 cfi_rel_offset (REG, 0) 73 74#define CFI_POP(REG) \ 75 cfi_adjust_cfa_offset (-4); \ 76 cfi_restore (REG) 77 78#define PUSH(REG) pushl REG; CFI_PUSH (REG) 79#define POP(REG) popl REG; CFI_POP (REG) 80 81#define PARMS 8 82#define ENTRANCE PUSH(%edi); 83#define RETURN POP(%edi); ret; CFI_PUSH(%edi); 84 85#define STR1 PARMS 86#define STR2 STR1+4 87 88 .text 89ENTRY (wcsrchr) 90 91 ENTRANCE 92 mov STR1(%esp), %ecx 93 movd STR2(%esp), %xmm1 94 95 mov %ecx, %edi 96 punpckldq %xmm1, %xmm1 97 pxor %xmm2, %xmm2 98 punpckldq %xmm1, %xmm1 99 100/* ECX has OFFSET. */ 101 and $63, %ecx 102 cmp $48, %ecx 103 ja L(crosscache) 104 105/* unaligned string. */ 106 movdqu (%edi), %xmm0 107 pcmpeqd %xmm0, %xmm2 108 pcmpeqd %xmm1, %xmm0 109/* Find where NULL is. */ 110 pmovmskb %xmm2, %ecx 111/* Check if there is a match. */ 112 pmovmskb %xmm0, %eax 113 add $16, %edi 114 115 test %eax, %eax 116 jnz L(unaligned_match1) 117 118 test %ecx, %ecx 119 jnz L(return_null) 120 121 and $-16, %edi 122 123 PUSH (%esi) 124 125 xor %edx, %edx 126 jmp L(loop) 127 128 CFI_POP (%esi) 129 130 .p2align 4 131L(unaligned_match1): 132 test %ecx, %ecx 133 jnz L(prolog_find_zero_1) 134 135 PUSH (%esi) 136 137/* Save current match */ 138 mov %eax, %edx 139 mov %edi, %esi 140 and $-16, %edi 141 jmp L(loop) 142 143 CFI_POP (%esi) 144 145 .p2align 4 146L(crosscache): 147/* Hancle unaligned string. */ 148 and $15, %ecx 149 and $-16, %edi 150 pxor %xmm3, %xmm3 151 movdqa (%edi), %xmm0 152 pcmpeqd %xmm0, %xmm3 153 pcmpeqd %xmm1, %xmm0 154/* Find where NULL is. */ 155 pmovmskb %xmm3, %edx 156/* Check if there is a match. */ 157 pmovmskb %xmm0, %eax 158/* Remove the leading bytes. */ 159 shr %cl, %edx 160 shr %cl, %eax 161 add $16, %edi 162 163 test %eax, %eax 164 jnz L(unaligned_match) 165 166 test %edx, %edx 167 jnz L(return_null) 168 169 PUSH (%esi) 170 171 xor %edx, %edx 172 jmp L(loop) 173 174 CFI_POP (%esi) 175 176 .p2align 4 177L(unaligned_match): 178 test %edx, %edx 179 jnz L(prolog_find_zero) 180 181 PUSH (%esi) 182 183 mov %eax, %edx 184 lea (%edi, %ecx), %esi 185 186/* Loop start on aligned string. */ 187 .p2align 4 188L(loop): 189 movdqa (%edi), %xmm0 190 pcmpeqd %xmm0, %xmm2 191 add $16, %edi 192 pcmpeqd %xmm1, %xmm0 193 pmovmskb %xmm2, %ecx 194 pmovmskb %xmm0, %eax 195 or %eax, %ecx 196 jnz L(matches) 197 198 movdqa (%edi), %xmm3 199 pcmpeqd %xmm3, %xmm2 200 add $16, %edi 201 pcmpeqd %xmm1, %xmm3 202 pmovmskb %xmm2, %ecx 203 pmovmskb %xmm3, %eax 204 or %eax, %ecx 205 jnz L(matches) 206 207 movdqa (%edi), %xmm4 208 pcmpeqd %xmm4, %xmm2 209 add $16, %edi 210 pcmpeqd %xmm1, %xmm4 211 pmovmskb %xmm2, %ecx 212 pmovmskb %xmm4, %eax 213 or %eax, %ecx 214 jnz L(matches) 215 216 movdqa (%edi), %xmm5 217 pcmpeqd %xmm5, %xmm2 218 add $16, %edi 219 pcmpeqd %xmm1, %xmm5 220 pmovmskb %xmm2, %ecx 221 pmovmskb %xmm5, %eax 222 or %eax, %ecx 223 jz L(loop) 224 225 .p2align 4 226L(matches): 227 test %eax, %eax 228 jnz L(match) 229L(return_value): 230 test %edx, %edx 231 jz L(return_null_1) 232 mov %edx, %eax 233 mov %esi, %edi 234 235 POP (%esi) 236 237 test %ah, %ah 238 jnz L(match_third_or_fourth_wchar) 239 test $15 << 4, %al 240 jnz L(match_second_wchar) 241 lea -16(%edi), %eax 242 RETURN 243 244 CFI_PUSH (%esi) 245 246 .p2align 4 247L(return_null_1): 248 POP (%esi) 249 250 xor %eax, %eax 251 RETURN 252 253 CFI_PUSH (%esi) 254 255 .p2align 4 256L(match): 257 pmovmskb %xmm2, %ecx 258 test %ecx, %ecx 259 jnz L(find_zero) 260/* save match info */ 261 mov %eax, %edx 262 mov %edi, %esi 263 jmp L(loop) 264 265 .p2align 4 266L(find_zero): 267 test %cl, %cl 268 jz L(find_zero_in_third_or_fourth_wchar) 269 test $15, %cl 270 jz L(find_zero_in_second_wchar) 271 and $1, %eax 272 jz L(return_value) 273 274 POP (%esi) 275 276 lea -16(%edi), %eax 277 RETURN 278 279 CFI_PUSH (%esi) 280 281 .p2align 4 282L(find_zero_in_second_wchar): 283 and $(1 << 5) - 1, %eax 284 jz L(return_value) 285 286 POP (%esi) 287 288 test $15 << 4, %al 289 jnz L(match_second_wchar) 290 lea -16(%edi), %eax 291 RETURN 292 293 CFI_PUSH (%esi) 294 295 .p2align 4 296L(find_zero_in_third_or_fourth_wchar): 297 test $15, %ch 298 jz L(find_zero_in_fourth_wchar) 299 and $(1 << 9) - 1, %eax 300 jz L(return_value) 301 302 POP (%esi) 303 304 test %ah, %ah 305 jnz L(match_third_wchar) 306 test $15 << 4, %al 307 jnz L(match_second_wchar) 308 lea -16(%edi), %eax 309 RETURN 310 311 CFI_PUSH (%esi) 312 313 .p2align 4 314L(find_zero_in_fourth_wchar): 315 316 POP (%esi) 317 318 test %ah, %ah 319 jnz L(match_third_or_fourth_wchar) 320 test $15 << 4, %al 321 jnz L(match_second_wchar) 322 lea -16(%edi), %eax 323 RETURN 324 325 CFI_PUSH (%esi) 326 327 .p2align 4 328L(match_second_wchar): 329 lea -12(%edi), %eax 330 RETURN 331 332 .p2align 4 333L(match_third_or_fourth_wchar): 334 test $15 << 4, %ah 335 jnz L(match_fourth_wchar) 336 lea -8(%edi), %eax 337 RETURN 338 339 .p2align 4 340L(match_third_wchar): 341 lea -8(%edi), %eax 342 RETURN 343 344 .p2align 4 345L(match_fourth_wchar): 346 lea -4(%edi), %eax 347 RETURN 348 349 .p2align 4 350L(return_null): 351 xor %eax, %eax 352 RETURN 353 354 .p2align 4 355L(prolog_find_zero): 356 add %ecx, %edi 357 mov %edx, %ecx 358L(prolog_find_zero_1): 359 test %cl, %cl 360 jz L(prolog_find_zero_in_third_or_fourth_wchar) 361 test $15, %cl 362 jz L(prolog_find_zero_in_second_wchar) 363 and $1, %eax 364 jz L(return_null) 365 366 lea -16(%edi), %eax 367 RETURN 368 369 .p2align 4 370L(prolog_find_zero_in_second_wchar): 371 and $(1 << 5) - 1, %eax 372 jz L(return_null) 373 374 test $15 << 4, %al 375 jnz L(match_second_wchar) 376 lea -16(%edi), %eax 377 RETURN 378 379 .p2align 4 380L(prolog_find_zero_in_third_or_fourth_wchar): 381 test $15, %ch 382 jz L(prolog_find_zero_in_fourth_wchar) 383 and $(1 << 9) - 1, %eax 384 jz L(return_null) 385 386 test %ah, %ah 387 jnz L(match_third_wchar) 388 test $15 << 4, %al 389 jnz L(match_second_wchar) 390 lea -16(%edi), %eax 391 RETURN 392 393 .p2align 4 394L(prolog_find_zero_in_fourth_wchar): 395 test %ah, %ah 396 jnz L(match_third_or_fourth_wchar) 397 test $15 << 4, %al 398 jnz L(match_second_wchar) 399 lea -16(%edi), %eax 400 RETURN 401 402END (wcsrchr) 403