1 // © 2016 and later: Unicode, Inc. and others.
2 // License & terms of use: http://www.unicode.org/copyright.html
3 /*
4  **********************************************************************
5  *   Copyright (C) 2005-2016, International Business Machines
6  *   Corporation and others.  All Rights Reserved.
7  **********************************************************************
8  */
9 
10 #include "unicode/utypes.h"
11 
12 #if !UCONFIG_NO_CONVERSION
13 
14 #include "unicode/ucsdet.h"
15 
16 #include "csdetect.h"
17 #include "csmatch.h"
18 #include "uenumimp.h"
19 
20 #include "cmemory.h"
21 #include "cstring.h"
22 #include "umutex.h"
23 #include "ucln_in.h"
24 #include "uarrsort.h"
25 #include "inputext.h"
26 #include "csrsbcs.h"
27 #include "csrmbcs.h"
28 #include "csrutf8.h"
29 #include "csrucode.h"
30 #include "csr2022.h"
31 
32 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
33 #define DELETE_ARRAY(array) uprv_free((void *) (array))
34 
35 U_NAMESPACE_BEGIN
36 
37 struct CSRecognizerInfo : public UMemory {
CSRecognizerInfoCSRecognizerInfo38     CSRecognizerInfo(CharsetRecognizer *recognizer, UBool isDefaultEnabled)
39         : recognizer(recognizer), isDefaultEnabled(isDefaultEnabled) {};
40 
~CSRecognizerInfoCSRecognizerInfo41     ~CSRecognizerInfo() {delete recognizer;};
42 
43     CharsetRecognizer *recognizer;
44     UBool isDefaultEnabled;
45 };
46 
47 U_NAMESPACE_END
48 
49 static icu::CSRecognizerInfo **fCSRecognizers = NULL;
50 static icu::UInitOnce gCSRecognizersInitOnce;
51 static int32_t fCSRecognizers_size = 0;
52 
53 U_CDECL_BEGIN
csdet_cleanup(void)54 static UBool U_CALLCONV csdet_cleanup(void)
55 {
56     U_NAMESPACE_USE
57     if (fCSRecognizers != NULL) {
58         for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
59             delete fCSRecognizers[r];
60             fCSRecognizers[r] = NULL;
61         }
62 
63         DELETE_ARRAY(fCSRecognizers);
64         fCSRecognizers = NULL;
65         fCSRecognizers_size = 0;
66     }
67     gCSRecognizersInitOnce.reset();
68 
69     return TRUE;
70 }
71 
72 static int32_t U_CALLCONV
charsetMatchComparator(const void *,const void * left,const void * right)73 charsetMatchComparator(const void * /*context*/, const void *left, const void *right)
74 {
75     U_NAMESPACE_USE
76 
77     const CharsetMatch **csm_l = (const CharsetMatch **) left;
78     const CharsetMatch **csm_r = (const CharsetMatch **) right;
79 
80     // NOTE: compare is backwards to sort from highest to lowest.
81     return (*csm_r)->getConfidence() - (*csm_l)->getConfidence();
82 }
83 
initRecognizers(UErrorCode & status)84 static void U_CALLCONV initRecognizers(UErrorCode &status) {
85     U_NAMESPACE_USE
86     ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
87     CSRecognizerInfo *tempArray[] = {
88         new CSRecognizerInfo(new CharsetRecog_UTF8(), TRUE),
89 
90         new CSRecognizerInfo(new CharsetRecog_UTF_16_BE(), TRUE),
91         new CSRecognizerInfo(new CharsetRecog_UTF_16_LE(), TRUE),
92         new CSRecognizerInfo(new CharsetRecog_UTF_32_BE(), TRUE),
93         new CSRecognizerInfo(new CharsetRecog_UTF_32_LE(), TRUE),
94 
95         new CSRecognizerInfo(new CharsetRecog_8859_1(), TRUE),
96         new CSRecognizerInfo(new CharsetRecog_8859_2(), TRUE),
97         new CSRecognizerInfo(new CharsetRecog_8859_5_ru(), TRUE),
98         new CSRecognizerInfo(new CharsetRecog_8859_6_ar(), TRUE),
99         new CSRecognizerInfo(new CharsetRecog_8859_7_el(), TRUE),
100         new CSRecognizerInfo(new CharsetRecog_8859_8_I_he(), TRUE),
101         new CSRecognizerInfo(new CharsetRecog_8859_8_he(), TRUE),
102         new CSRecognizerInfo(new CharsetRecog_windows_1251(), TRUE),
103         new CSRecognizerInfo(new CharsetRecog_windows_1256(), TRUE),
104         new CSRecognizerInfo(new CharsetRecog_KOI8_R(), TRUE),
105         new CSRecognizerInfo(new CharsetRecog_8859_9_tr(), TRUE),
106         new CSRecognizerInfo(new CharsetRecog_sjis(), TRUE),
107         new CSRecognizerInfo(new CharsetRecog_gb_18030(), TRUE),
108         new CSRecognizerInfo(new CharsetRecog_euc_jp(), TRUE),
109         new CSRecognizerInfo(new CharsetRecog_euc_kr(), TRUE),
110         new CSRecognizerInfo(new CharsetRecog_big5(), TRUE),
111 
112         new CSRecognizerInfo(new CharsetRecog_2022JP(), TRUE),
113 #if !UCONFIG_ONLY_HTML_CONVERSION
114         new CSRecognizerInfo(new CharsetRecog_2022KR(), TRUE),
115         new CSRecognizerInfo(new CharsetRecog_2022CN(), TRUE),
116 
117         new CSRecognizerInfo(new CharsetRecog_IBM424_he_rtl(), FALSE),
118         new CSRecognizerInfo(new CharsetRecog_IBM424_he_ltr(), FALSE),
119         new CSRecognizerInfo(new CharsetRecog_IBM420_ar_rtl(), FALSE),
120         new CSRecognizerInfo(new CharsetRecog_IBM420_ar_ltr(), FALSE)
121 #endif
122     };
123     int32_t rCount = UPRV_LENGTHOF(tempArray);
124 
125     fCSRecognizers = NEW_ARRAY(CSRecognizerInfo *, rCount);
126 
127     if (fCSRecognizers == NULL) {
128         status = U_MEMORY_ALLOCATION_ERROR;
129     }
130     else {
131         fCSRecognizers_size = rCount;
132         for (int32_t r = 0; r < rCount; r += 1) {
133             fCSRecognizers[r] = tempArray[r];
134             if (fCSRecognizers[r] == NULL) {
135                 status = U_MEMORY_ALLOCATION_ERROR;
136             }
137         }
138     }
139 }
140 
141 U_CDECL_END
142 
143 U_NAMESPACE_BEGIN
144 
setRecognizers(UErrorCode & status)145 void CharsetDetector::setRecognizers(UErrorCode &status)
146 {
147     umtx_initOnce(gCSRecognizersInitOnce, &initRecognizers, status);
148 }
149 
CharsetDetector(UErrorCode & status)150 CharsetDetector::CharsetDetector(UErrorCode &status)
151   : textIn(new InputText(status)), resultArray(NULL),
152     resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE),
153     fEnabledRecognizers(NULL)
154 {
155     if (U_FAILURE(status)) {
156         return;
157     }
158 
159     setRecognizers(status);
160 
161     if (U_FAILURE(status)) {
162         return;
163     }
164 
165     resultArray = (CharsetMatch **)uprv_malloc(sizeof(CharsetMatch *)*fCSRecognizers_size);
166 
167     if (resultArray == NULL) {
168         status = U_MEMORY_ALLOCATION_ERROR;
169         return;
170     }
171 
172     for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
173         resultArray[i] = new CharsetMatch();
174 
175         if (resultArray[i] == NULL) {
176             status = U_MEMORY_ALLOCATION_ERROR;
177             break;
178         }
179     }
180 }
181 
~CharsetDetector()182 CharsetDetector::~CharsetDetector()
183 {
184     delete textIn;
185 
186     for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
187         delete resultArray[i];
188     }
189 
190     uprv_free(resultArray);
191 
192     if (fEnabledRecognizers) {
193         uprv_free(fEnabledRecognizers);
194     }
195 }
196 
setText(const char * in,int32_t len)197 void CharsetDetector::setText(const char *in, int32_t len)
198 {
199     textIn->setText(in, len);
200     fFreshTextSet = TRUE;
201 }
202 
setStripTagsFlag(UBool flag)203 UBool CharsetDetector::setStripTagsFlag(UBool flag)
204 {
205     UBool temp = fStripTags;
206     fStripTags = flag;
207     fFreshTextSet = TRUE;
208     return temp;
209 }
210 
getStripTagsFlag() const211 UBool CharsetDetector::getStripTagsFlag() const
212 {
213     return fStripTags;
214 }
215 
setDeclaredEncoding(const char * encoding,int32_t len) const216 void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const
217 {
218     textIn->setDeclaredEncoding(encoding,len);
219 }
220 
getDetectableCount()221 int32_t CharsetDetector::getDetectableCount()
222 {
223     UErrorCode status = U_ZERO_ERROR;
224 
225     setRecognizers(status);
226 
227     return fCSRecognizers_size;
228 }
229 
detect(UErrorCode & status)230 const CharsetMatch *CharsetDetector::detect(UErrorCode &status)
231 {
232     int32_t maxMatchesFound = 0;
233 
234     detectAll(maxMatchesFound, status);
235 
236     if(maxMatchesFound > 0) {
237         return resultArray[0];
238     } else {
239         return NULL;
240     }
241 }
242 
detectAll(int32_t & maxMatchesFound,UErrorCode & status)243 const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status)
244 {
245     if(!textIn->isSet()) {
246         status = U_MISSING_RESOURCE_ERROR;// TODO:  Need to set proper status code for input text not set
247 
248         return NULL;
249     } else if (fFreshTextSet) {
250         CharsetRecognizer *csr;
251         int32_t            i;
252 
253         textIn->MungeInput(fStripTags);
254 
255         // Iterate over all possible charsets, remember all that
256         // give a match quality > 0.
257         resultCount = 0;
258         for (i = 0; i < fCSRecognizers_size; i += 1) {
259             csr = fCSRecognizers[i]->recognizer;
260             if (csr->match(textIn, resultArray[resultCount])) {
261                 resultCount++;
262             }
263         }
264 
265         if (resultCount > 1) {
266             uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status);
267         }
268         fFreshTextSet = FALSE;
269     }
270 
271     maxMatchesFound = resultCount;
272 
273     return resultArray;
274 }
275 
setDetectableCharset(const char * encoding,UBool enabled,UErrorCode & status)276 void CharsetDetector::setDetectableCharset(const char *encoding, UBool enabled, UErrorCode &status)
277 {
278     if (U_FAILURE(status)) {
279         return;
280     }
281 
282     int32_t modIdx = -1;
283     UBool isDefaultVal = FALSE;
284     for (int32_t i = 0; i < fCSRecognizers_size; i++) {
285         CSRecognizerInfo *csrinfo = fCSRecognizers[i];
286         if (uprv_strcmp(csrinfo->recognizer->getName(), encoding) == 0) {
287             modIdx = i;
288             isDefaultVal = (csrinfo->isDefaultEnabled == enabled);
289             break;
290         }
291     }
292     if (modIdx < 0) {
293         // No matching encoding found
294         status = U_ILLEGAL_ARGUMENT_ERROR;
295         return;
296     }
297 
298     if (fEnabledRecognizers == NULL && !isDefaultVal) {
299         // Create an array storing the non default setting
300         fEnabledRecognizers = NEW_ARRAY(UBool, fCSRecognizers_size);
301         if (fEnabledRecognizers == NULL) {
302             status = U_MEMORY_ALLOCATION_ERROR;
303             return;
304         }
305         // Initialize the array with default info
306         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
307             fEnabledRecognizers[i] = fCSRecognizers[i]->isDefaultEnabled;
308         }
309     }
310 
311     if (fEnabledRecognizers != NULL) {
312         fEnabledRecognizers[modIdx] = enabled;
313     }
314 }
315 
316 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
317 {
318     if( index > fCSRecognizers_size-1 || index < 0) {
319         status = U_INDEX_OUTOFBOUNDS_ERROR;
320 
321         return 0;
322     } else {
323         return fCSRecognizers[index]->getName();
324     }
325 }*/
326 
327 U_NAMESPACE_END
328 
329 U_CDECL_BEGIN
330 typedef struct {
331     int32_t currIndex;
332     UBool all;
333     UBool *enabledRecognizers;
334 } Context;
335 
336 
337 
338 static void U_CALLCONV
enumClose(UEnumeration * en)339 enumClose(UEnumeration *en) {
340     if(en->context != NULL) {
341         DELETE_ARRAY(en->context);
342     }
343 
344     DELETE_ARRAY(en);
345 }
346 
347 static int32_t U_CALLCONV
enumCount(UEnumeration * en,UErrorCode *)348 enumCount(UEnumeration *en, UErrorCode *) {
349     if (((Context *)en->context)->all) {
350         // ucsdet_getAllDetectableCharsets, all charset detector names
351         return fCSRecognizers_size;
352     }
353 
354     // Otherwise, ucsdet_getDetectableCharsets - only enabled ones
355     int32_t count = 0;
356     UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
357     if (enabledArray != NULL) {
358         // custom set
359         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
360             if (enabledArray[i]) {
361                 count++;
362             }
363         }
364     } else {
365         // default set
366         for (int32_t i = 0; i < fCSRecognizers_size; i++) {
367             if (fCSRecognizers[i]->isDefaultEnabled) {
368                 count++;
369             }
370         }
371     }
372     return count;
373 }
374 
375 static const char* U_CALLCONV
enumNext(UEnumeration * en,int32_t * resultLength,UErrorCode *)376 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) {
377     const char *currName = NULL;
378 
379     if (((Context *)en->context)->currIndex < fCSRecognizers_size) {
380         if (((Context *)en->context)->all) {
381             // ucsdet_getAllDetectableCharsets, all charset detector names
382             currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
383             ((Context *)en->context)->currIndex++;
384         } else {
385             // ucsdet_getDetectableCharsets
386             UBool *enabledArray = ((Context *)en->context)->enabledRecognizers;
387             if (enabledArray != NULL) {
388                 // custome set
389                 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
390                     if (enabledArray[((Context *)en->context)->currIndex]) {
391                         currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
392                     }
393                     ((Context *)en->context)->currIndex++;
394                 }
395             } else {
396                 // default set
397                 while (currName == NULL && ((Context *)en->context)->currIndex < fCSRecognizers_size) {
398                     if (fCSRecognizers[((Context *)en->context)->currIndex]->isDefaultEnabled) {
399                         currName = fCSRecognizers[((Context *)en->context)->currIndex]->recognizer->getName();
400                     }
401                     ((Context *)en->context)->currIndex++;
402                 }
403             }
404         }
405     }
406 
407     if(resultLength != NULL) {
408         *resultLength = currName == NULL ? 0 : (int32_t)uprv_strlen(currName);
409     }
410 
411     return currName;
412 }
413 
414 
415 static void U_CALLCONV
enumReset(UEnumeration * en,UErrorCode *)416 enumReset(UEnumeration *en, UErrorCode *) {
417     ((Context *)en->context)->currIndex = 0;
418 }
419 
420 static const UEnumeration gCSDetEnumeration = {
421     NULL,
422     NULL,
423     enumClose,
424     enumCount,
425     uenum_unextDefault,
426     enumNext,
427     enumReset
428 };
429 
430 U_CDECL_END
431 
432 U_NAMESPACE_BEGIN
433 
getAllDetectableCharsets(UErrorCode & status)434 UEnumeration * CharsetDetector::getAllDetectableCharsets(UErrorCode &status)
435 {
436 
437     /* Initialize recognized charsets. */
438     setRecognizers(status);
439 
440     if(U_FAILURE(status)) {
441         return 0;
442     }
443 
444     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
445     if (en == NULL) {
446         status = U_MEMORY_ALLOCATION_ERROR;
447         return 0;
448     }
449     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
450     en->context = (void*)NEW_ARRAY(Context, 1);
451     if (en->context == NULL) {
452         status = U_MEMORY_ALLOCATION_ERROR;
453         DELETE_ARRAY(en);
454         return 0;
455     }
456     uprv_memset(en->context, 0, sizeof(Context));
457     ((Context*)en->context)->all = TRUE;
458     return en;
459 }
460 
getDetectableCharsets(UErrorCode & status) const461 UEnumeration * CharsetDetector::getDetectableCharsets(UErrorCode &status) const
462 {
463     if(U_FAILURE(status)) {
464         return 0;
465     }
466 
467     UEnumeration *en = NEW_ARRAY(UEnumeration, 1);
468     if (en == NULL) {
469         status = U_MEMORY_ALLOCATION_ERROR;
470         return 0;
471     }
472     memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration));
473     en->context = (void*)NEW_ARRAY(Context, 1);
474     if (en->context == NULL) {
475         status = U_MEMORY_ALLOCATION_ERROR;
476         DELETE_ARRAY(en);
477         return 0;
478     }
479     uprv_memset(en->context, 0, sizeof(Context));
480     ((Context*)en->context)->all = FALSE;
481     ((Context*)en->context)->enabledRecognizers = fEnabledRecognizers;
482     return en;
483 }
484 
485 U_NAMESPACE_END
486 
487 #endif
488