1 // © 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 2000-2015, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * 11 * File reslist.h 12 * 13 * Modification History: 14 * 15 * Date Name Description 16 * 02/21/00 weiv Creation. 17 ******************************************************************************* 18 */ 19 20 #ifndef RESLIST_H 21 #define RESLIST_H 22 23 #define KEY_SPACE_SIZE 65536 24 #define RESLIST_MAX_INT_VECTOR 2048 25 26 #include <functional> 27 28 #include "unicode/utypes.h" 29 #include "unicode/unistr.h" 30 #include "unicode/ures.h" 31 #include "unicode/ustring.h" 32 #include "cmemory.h" 33 #include "cstring.h" 34 #include "uhash.h" 35 #include "unewdata.h" 36 #include "uresdata.h" 37 #include "ustr.h" 38 39 U_CDECL_BEGIN 40 41 class PathFilter; 42 class PseudoListResource; 43 class ResKeyPath; 44 45 struct ResFile { ResFileResFile46 ResFile() 47 : fBytes(NULL), fIndexes(NULL), 48 fKeys(NULL), fKeysLength(0), fKeysCount(0), 49 fStrings(NULL), fStringIndexLimit(0), 50 fChecksum(0) {} ~ResFileResFile51 ~ResFile() { close(); } 52 53 void close(); 54 55 uint8_t *fBytes; 56 const int32_t *fIndexes; 57 const char *fKeys; 58 int32_t fKeysLength; 59 int32_t fKeysCount; 60 61 PseudoListResource *fStrings; 62 int32_t fStringIndexLimit; 63 64 int32_t fChecksum; 65 }; 66 67 struct SResource; 68 69 typedef struct KeyMapEntry { 70 int32_t oldpos, newpos; 71 } KeyMapEntry; 72 73 /* Resource bundle root table */ 74 struct SRBRoot { 75 SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCode); 76 ~SRBRoot(); 77 78 void write(const char *outputDir, const char *outputPkg, 79 char *writtenFilename, int writtenFilenameLen, UErrorCode &errorCode); 80 81 void setLocale(UChar *locale, UErrorCode &errorCode); 82 int32_t addTag(const char *tag, UErrorCode &errorCode); 83 84 const char *getKeyString(int32_t key) const; 85 const char *getKeyBytes(int32_t *pLength) const; 86 87 int32_t addKeyBytes(const char *keyBytes, int32_t length, UErrorCode &errorCode); 88 89 void compactKeys(UErrorCode &errorCode); 90 91 int32_t makeRes16(uint32_t resWord) const; 92 int32_t mapKey(int32_t oldpos) const; 93 94 private: 95 void compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode); 96 97 public: 98 // TODO: private 99 100 SResource *fRoot; // Normally a TableResource. 101 char *fLocale; 102 int32_t fIndexLength; 103 int32_t fMaxTableLength; 104 UBool fNoFallback; /* see URES_ATT_NO_FALLBACK */ 105 int8_t fStringsForm; /* default STRINGS_UTF16_V1 */ 106 UBool fIsPoolBundle; 107 108 char *fKeys; 109 KeyMapEntry *fKeyMap; 110 int32_t fKeysBottom, fKeysTop; 111 int32_t fKeysCapacity; 112 int32_t fKeysCount; 113 int32_t fLocalKeyLimit; /* key offset < limit fits into URES_TABLE */ 114 115 icu::UnicodeString f16BitUnits; 116 int32_t f16BitStringsLength; 117 118 const ResFile *fUsePoolBundle; 119 int32_t fPoolStringIndexLimit; 120 int32_t fPoolStringIndex16Limit; 121 int32_t fLocalStringIndexLimit; 122 SRBRoot *fWritePoolBundle; 123 }; 124 125 /* write a java resource file */ 126 // TODO: C++ify 127 void bundle_write_java(struct SRBRoot *bundle, const char *outputDir, const char* outputEnc, char *writtenFilename, 128 int writtenFilenameLen, const char* packageName, const char* bundleName, UErrorCode *status); 129 130 /* write a xml resource file */ 131 // TODO: C++ify 132 void bundle_write_xml(struct SRBRoot *bundle, const char *outputDir,const char* outputEnc, const char* rbname, 133 char *writtenFilename, int writtenFilenameLen, const char* language, const char* package, UErrorCode *status); 134 135 /* Various resource types */ 136 137 /* 138 * Return a unique pointer to a dummy object, 139 * for use in non-error cases when no resource is to be added to the bundle. 140 * (NULL is used in error cases.) 141 */ 142 struct SResource* res_none(void); 143 144 class ArrayResource; 145 class TableResource; 146 class IntVectorResource; 147 148 TableResource *table_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status); 149 150 ArrayResource *array_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status); 151 152 struct SResource *string_open(struct SRBRoot *bundle, const char *tag, const UChar *value, int32_t len, const struct UString* comment, UErrorCode *status); 153 154 struct SResource *alias_open(struct SRBRoot *bundle, const char *tag, UChar *value, int32_t len, const struct UString* comment, UErrorCode *status); 155 156 IntVectorResource *intvector_open(struct SRBRoot *bundle, const char *tag, const struct UString* comment, UErrorCode *status); 157 158 struct SResource *int_open(struct SRBRoot *bundle, const char *tag, int32_t value, const struct UString* comment, UErrorCode *status); 159 160 struct SResource *bin_open(struct SRBRoot *bundle, const char *tag, uint32_t length, uint8_t *data, const char* fileName, const struct UString* comment, UErrorCode *status); 161 162 /* Resource place holder */ 163 164 struct SResource { 165 SResource(); 166 SResource(SRBRoot *bundle, const char *tag, int8_t type, const UString* comment, 167 UErrorCode &errorCode); 168 virtual ~SResource(); 169 isTableSResource170 UBool isTable() const { return fType == URES_TABLE; } isStringSResource171 UBool isString() const { return fType == URES_STRING; } 172 173 const char *getKeyString(const SRBRoot *bundle) const; 174 175 /** 176 * Preflights strings. 177 * Finds duplicates and counts the total number of string code units 178 * so that they can be written first to the 16-bit array, 179 * for minimal string and container storage. 180 * 181 * We walk the final parse tree, rather than collecting this information while building it, 182 * so that we need not deal with changes to the parse tree (especially removing resources). 183 */ 184 void preflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode); 185 virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode); 186 187 /** 188 * Writes resource values into f16BitUnits 189 * and determines the resource item word, if possible. 190 */ 191 void write16(SRBRoot *bundle); 192 virtual void handleWrite16(SRBRoot *bundle); 193 194 /** 195 * Calculates ("preflights") and advances the *byteOffset 196 * by the size of the resource's data in the binary file and 197 * determines the resource item word. 198 * 199 * Most handlePreWrite() functions may add any number of bytes, but preWrite() 200 * will always pad it to a multiple of 4. 201 * The resource item type may be a related subtype of the fType. 202 * 203 * The preWrite() and write() functions start and end at the same 204 * byteOffset values. 205 * Prewriting allows bundle.write() to determine the root resource item word, 206 * before actually writing the bundle contents to the file, 207 * which is necessary because the root item is stored at the beginning. 208 */ 209 void preWrite(uint32_t *byteOffset); 210 virtual void handlePreWrite(uint32_t *byteOffset); 211 212 /** 213 * Writes the resource's data to mem and updates the byteOffset 214 * in parallel. 215 */ 216 void write(UNewDataMemory *mem, uint32_t *byteOffset); 217 virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset); 218 219 /** 220 * Applies the given filter with the given base path to this resource. 221 * Removes child resources rejected by the filter recursively. 222 * 223 * @param bundle Needed in order to access the key for this and child resources. 224 */ 225 virtual void applyFilter(const PathFilter& filter, ResKeyPath& path, const SRBRoot* bundle); 226 227 /** 228 * Calls the given function for every key ID present in this tree. 229 */ 230 virtual void collectKeys(std::function<void(int32_t)> collector) const; 231 232 int8_t fType; /* nominal type: fRes (when != 0xffffffff) may use subtype */ 233 UBool fWritten; /* res_write() can exit early */ 234 uint32_t fRes; /* resource item word; RES_BOGUS=0xffffffff if not known yet */ 235 int32_t fRes16; /* Res16 version of fRes for Table, Table16, Array16; -1 if it does not fit. */ 236 int32_t fKey; /* Index into bundle->fKeys; -1 if no key. */ 237 int32_t fKey16; /* Key16 version of fKey for Table & Table16; -1 if no key or it does not fit. */ 238 int line; /* used internally to report duplicate keys in tables */ 239 SResource *fNext; /* This is for internal chaining while building */ 240 struct UString fComment; 241 }; 242 243 class ContainerResource : public SResource { 244 public: ContainerResource(SRBRoot * bundle,const char * tag,int8_t type,const UString * comment,UErrorCode & errorCode)245 ContainerResource(SRBRoot *bundle, const char *tag, int8_t type, 246 const UString* comment, UErrorCode &errorCode) 247 : SResource(bundle, tag, type, comment, errorCode), 248 fCount(0), fFirst(NULL) {} 249 virtual ~ContainerResource(); 250 251 void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode) override; 252 253 void collectKeys(std::function<void(int32_t)> collector) const override; 254 255 protected: 256 void writeAllRes16(SRBRoot *bundle); 257 void preWriteAllRes(uint32_t *byteOffset); 258 void writeAllRes(UNewDataMemory *mem, uint32_t *byteOffset); 259 void writeAllRes32(UNewDataMemory *mem, uint32_t *byteOffset); 260 261 public: 262 // TODO: private with getter? 263 uint32_t fCount; 264 SResource *fFirst; 265 }; 266 267 class TableResource : public ContainerResource { 268 public: TableResource(SRBRoot * bundle,const char * tag,const UString * comment,UErrorCode & errorCode)269 TableResource(SRBRoot *bundle, const char *tag, 270 const UString* comment, UErrorCode &errorCode) 271 : ContainerResource(bundle, tag, URES_TABLE, comment, errorCode), 272 fTableType(URES_TABLE), fRoot(bundle) {} 273 virtual ~TableResource(); 274 275 void add(SResource *res, int linenumber, UErrorCode &errorCode); 276 277 void handleWrite16(SRBRoot *bundle) override; 278 void handlePreWrite(uint32_t *byteOffset) override; 279 void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) override; 280 281 void applyFilter(const PathFilter& filter, ResKeyPath& path, const SRBRoot* bundle) override; 282 283 int8_t fTableType; // determined by table_write16() for table_preWrite() & table_write() 284 SRBRoot *fRoot; 285 }; 286 287 class ArrayResource : public ContainerResource { 288 public: ArrayResource(SRBRoot * bundle,const char * tag,const UString * comment,UErrorCode & errorCode)289 ArrayResource(SRBRoot *bundle, const char *tag, 290 const UString* comment, UErrorCode &errorCode) 291 : ContainerResource(bundle, tag, URES_ARRAY, comment, errorCode), 292 fLast(NULL) {} 293 virtual ~ArrayResource(); 294 295 void add(SResource *res); 296 297 virtual void handleWrite16(SRBRoot *bundle); 298 virtual void handlePreWrite(uint32_t *byteOffset); 299 virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset); 300 301 SResource *fLast; 302 }; 303 304 /** 305 * List of resources for a pool bundle. 306 * Writes an empty table resource, rather than a container structure. 307 */ 308 class PseudoListResource : public ContainerResource { 309 public: PseudoListResource(SRBRoot * bundle,UErrorCode & errorCode)310 PseudoListResource(SRBRoot *bundle, UErrorCode &errorCode) 311 : ContainerResource(bundle, NULL, URES_TABLE, NULL, errorCode) {} 312 virtual ~PseudoListResource(); 313 314 void add(SResource *res); 315 316 virtual void handleWrite16(SRBRoot *bundle); 317 }; 318 319 class StringBaseResource : public SResource { 320 public: 321 StringBaseResource(SRBRoot *bundle, const char *tag, int8_t type, 322 const UChar *value, int32_t len, 323 const UString* comment, UErrorCode &errorCode); 324 StringBaseResource(SRBRoot *bundle, int8_t type, 325 const icu::UnicodeString &value, UErrorCode &errorCode); 326 StringBaseResource(int8_t type, const UChar *value, int32_t len, UErrorCode &errorCode); 327 virtual ~StringBaseResource(); 328 getBuffer()329 const UChar *getBuffer() const { return icu::toUCharPtr(fString.getBuffer()); } length()330 int32_t length() const { return fString.length(); } 331 332 virtual void handlePreWrite(uint32_t *byteOffset); 333 virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset); 334 335 // TODO: private with getter? 336 icu::UnicodeString fString; 337 }; 338 339 class StringResource : public StringBaseResource { 340 public: StringResource(SRBRoot * bundle,const char * tag,const UChar * value,int32_t len,const UString * comment,UErrorCode & errorCode)341 StringResource(SRBRoot *bundle, const char *tag, const UChar *value, int32_t len, 342 const UString* comment, UErrorCode &errorCode) 343 : StringBaseResource(bundle, tag, URES_STRING, value, len, comment, errorCode), 344 fSame(NULL), fSuffixOffset(0), 345 fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(0) {} StringResource(SRBRoot * bundle,const icu::UnicodeString & value,UErrorCode & errorCode)346 StringResource(SRBRoot *bundle, const icu::UnicodeString &value, UErrorCode &errorCode) 347 : StringBaseResource(bundle, URES_STRING, value, errorCode), 348 fSame(NULL), fSuffixOffset(0), 349 fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(0) {} StringResource(int32_t poolStringIndex,int8_t numCharsForLength,const UChar * value,int32_t length,UErrorCode & errorCode)350 StringResource(int32_t poolStringIndex, int8_t numCharsForLength, 351 const UChar *value, int32_t length, 352 UErrorCode &errorCode) 353 : StringBaseResource(URES_STRING, value, length, errorCode), 354 fSame(NULL), fSuffixOffset(0), 355 fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(numCharsForLength) { 356 // v3 pool string encoded as string-v2 with low offset 357 fRes = URES_MAKE_RESOURCE(URES_STRING_V2, poolStringIndex); 358 fWritten = true; 359 } 360 virtual ~StringResource(); 361 get16BitStringsLength()362 int32_t get16BitStringsLength() const { 363 return fNumCharsForLength + length() + 1; // +1 for the NUL 364 } 365 366 virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode); 367 virtual void handleWrite16(SRBRoot *bundle); 368 369 void writeUTF16v2(int32_t base, icu::UnicodeString &dest); 370 371 StringResource *fSame; // used for duplicates 372 int32_t fSuffixOffset; // this string is a suffix of fSame at this offset 373 int32_t fNumCopies; // number of equal strings represented by one stringSet element 374 int32_t fNumUnitsSaved; // from not writing duplicates and suffixes 375 int8_t fNumCharsForLength; 376 }; 377 378 class AliasResource : public StringBaseResource { 379 public: AliasResource(SRBRoot * bundle,const char * tag,const UChar * value,int32_t len,const UString * comment,UErrorCode & errorCode)380 AliasResource(SRBRoot *bundle, const char *tag, const UChar *value, int32_t len, 381 const UString* comment, UErrorCode &errorCode) 382 : StringBaseResource(bundle, tag, URES_ALIAS, value, len, comment, errorCode) {} 383 virtual ~AliasResource(); 384 }; 385 386 class IntResource : public SResource { 387 public: 388 IntResource(SRBRoot *bundle, const char *tag, int32_t value, 389 const UString* comment, UErrorCode &errorCode); 390 virtual ~IntResource(); 391 392 // TODO: private with getter? 393 int32_t fValue; 394 }; 395 396 class IntVectorResource : public SResource { 397 public: 398 IntVectorResource(SRBRoot *bundle, const char *tag, 399 const UString* comment, UErrorCode &errorCode); 400 virtual ~IntVectorResource(); 401 402 void add(int32_t value, UErrorCode &errorCode); 403 404 virtual void handlePreWrite(uint32_t *byteOffset); 405 virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset); 406 407 // TODO: UVector32 408 uint32_t fCount; 409 uint32_t *fArray; 410 }; 411 412 class BinaryResource : public SResource { 413 public: 414 BinaryResource(SRBRoot *bundle, const char *tag, 415 uint32_t length, uint8_t *data, const char* fileName, 416 const UString* comment, UErrorCode &errorCode); 417 virtual ~BinaryResource(); 418 419 virtual void handlePreWrite(uint32_t *byteOffset); 420 virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset); 421 422 // TODO: CharString? 423 uint32_t fLength; 424 uint8_t *fData; 425 // TODO: CharString 426 char* fFileName; // file name for binary or import binary tags if any 427 }; 428 429 // TODO: use LocalPointer or delete 430 void res_close(struct SResource *res); 431 432 void setIncludeCopyright(UBool val); 433 UBool getIncludeCopyright(void); 434 435 void setFormatVersion(int32_t formatVersion); 436 437 int32_t getFormatVersion(); 438 439 void setUsePoolBundle(UBool use); 440 441 /* in wrtxml.cpp */ 442 uint32_t computeCRC(const char *ptr, uint32_t len, uint32_t lastcrc); 443 444 U_CDECL_END 445 #endif /* #ifndef RESLIST_H */ 446