Lines Matching refs:TextIdx
9 typedef DurchschlagTextIdx TextIdx; typedef
15 TextIdx next;
16 TextIdx offset;
20 TextIdx mark;
25 TextIdx start;
26 TextIdx end;
31 TextIdx position;
55 static TextIdx calculateDictionarySize(const std::vector<Range>& ranges) { in calculateDictionarySize()
56 TextIdx result = 0; in calculateDictionarySize()
81 std::vector<MetaSlot>* map, TextIdx span, const TextIdx* shortcut, in buildCandidatesList()
82 TextIdx end) { in buildCandidatesList()
103 TextIdx i = 0; in buildCandidatesList()
104 TextIdx limit = std::min<TextIdx>(end, CANDIDATE_BUNDLE_SIZE); in buildCandidatesList()
107 TextIdx slice = shortcut[i + span - 1]; in buildCandidatesList()
129 TextIdx slice = shortcut[i + span - 1]; in buildCandidatesList()
168 static Score rebuildCandidatesList(std::vector<TextIdx>* candidates, in rebuildCandidatesList()
169 std::vector<MetaSlot>* map, TextIdx span, const TextIdx* shortcut, in rebuildCandidatesList()
170 TextIdx end, TextIdx* next) { in rebuildCandidatesList()
172 TextIdx* data = candidates->data(); in rebuildCandidatesList()
187 for (TextIdx i = 0; i < span - 1; ++i) { in rebuildCandidatesList()
196 for (TextIdx i = 0; i < end; ++i) { in rebuildCandidatesList()
227 static void addRange(std::vector<Range>* ranges, TextIdx start, TextIdx end) { in addRange()
258 TextIdx sliceLen = static_cast<TextIdx>(slice_len); in durchschlag_prepare()
263 TextIdx total = 0; in durchschlag_prepare()
264 std::vector<TextIdx> offsets; in durchschlag_prepare()
267 TextIdx delta = static_cast<TextIdx>(sample_sizes[i]); in durchschlag_prepare()
270 TextIdx next_total = total + delta; in durchschlag_prepare()
277 TextIdx end = total - static_cast<TextIdx>(sliceLen) + 1; in durchschlag_prepare()
278 TextIdx hashLen = 11; in durchschlag_prepare()
283 TextIdx hashMask = (1u << hashLen) - 1u; in durchschlag_prepare()
284 std::vector<TextIdx> hashHead(1 << hashLen); in durchschlag_prepare()
285 TextIdx hash = 0; in durchschlag_prepare()
286 TextIdx lShift = 3; in durchschlag_prepare()
287 TextIdx rShift = hashLen - lShift; in durchschlag_prepare()
288 for (TextIdx i = 0; i < sliceLen - 1; ++i) { in durchschlag_prepare()
289 TextIdx v = data[i]; in durchschlag_prepare()
292 TextIdx lShiftX = (lShift * (sliceLen - 1)) % hashLen; in durchschlag_prepare()
293 TextIdx rShiftX = hashLen - lShiftX; in durchschlag_prepare()
297 TextIdx hashSlot = 1; in durchschlag_prepare()
298 std::vector<TextIdx> sliceMap; in durchschlag_prepare()
300 for (TextIdx i = 0; i < end; ++i) { in durchschlag_prepare()
301 TextIdx v = data[i + sliceLen - 1]; in durchschlag_prepare()
302 TextIdx bucket = (((hash << lShift) | (hash >> rShift)) & hashMask) ^ v; in durchschlag_prepare()
305 TextIdx slot = hashHead[bucket]; in durchschlag_prepare()
308 TextIdx start = item.offset; in durchschlag_prepare()
310 for (TextIdx j = 0; j < sliceLen; ++j) { in durchschlag_prepare()
330 return {total, sliceLen, static_cast<TextIdx>(map.size()), in durchschlag_prepare()
337 TextIdx sliceLen = static_cast<TextIdx>(slice_len); in durchschlag_prepare()
340 const TextIdx* lcp = index.lcp.data(); in durchschlag_prepare()
341 const TextIdx* sa = index.sa.data(); in durchschlag_prepare()
343 TextIdx total = 0; in durchschlag_prepare()
344 std::vector<TextIdx> offsets; in durchschlag_prepare()
347 TextIdx delta = static_cast<TextIdx>(sample_sizes[i]); in durchschlag_prepare()
350 TextIdx next_total = total + delta; in durchschlag_prepare()
357 TextIdx counter = 1; in durchschlag_prepare()
358 TextIdx end = total - sliceLen + 1; in durchschlag_prepare()
359 std::vector<TextIdx> sliceMap(total); in durchschlag_prepare()
360 TextIdx last = 0; in durchschlag_prepare()
361 TextIdx current = 1; in durchschlag_prepare()
364 for (TextIdx i = last; i < current; ++i) { in durchschlag_prepare()
375 std::vector<TextIdx> reorder(counter); in durchschlag_prepare()
377 for (TextIdx i = 0; i < end; ++i) { in durchschlag_prepare()
382 for (TextIdx i = 0; i < end; ++i) { in durchschlag_prepare()
390 TextIdx total = static_cast<TextIdx>(data.size()); in durchschlag_index()
394 if (static_cast<TextIdx>(saTotal) != total) fatal("corpus is too large"); in durchschlag_index()
395 std::vector<TextIdx> sa(total); in durchschlag_index()
397 if (sizeof(TextIdx) != sizeof(int32_t)) fatal("type length mismatch"); in durchschlag_index()
401 std::vector<TextIdx> isa(total); in durchschlag_index()
402 for (TextIdx i = 0; i < total; ++i) isa[sa[i]] = i; in durchschlag_index()
405 std::vector<TextIdx> lcp(total); in durchschlag_index()
406 TextIdx k = 0; in durchschlag_index()
408 for (TextIdx i = 0; i < total; ++i) { in durchschlag_index()
409 TextIdx current = isa[i]; in durchschlag_index()
414 TextIdx j = sa[current + 1]; // Suffix which follow i-th suffix. in durchschlag_index()
425 static void ScoreSlices(const std::vector<TextIdx>& offsets, in ScoreSlices()
426 std::vector<MetaSlot>& map, const TextIdx* shortcut, TextIdx end) { in ScoreSlices()
427 TextIdx piece = 0; in ScoreSlices()
429 TextIdx mark = 1; in ScoreSlices()
430 for (TextIdx i = 0; i < end; ++i) { in ScoreSlices()
447 TextIdx targetSize = static_cast<TextIdx>(dictionary_size_limit); in durchschlagGenerateExclusive()
452 TextIdx sliceLen = context.sliceLen; in durchschlagGenerateExclusive()
453 TextIdx total = context.dataSize; in durchschlagGenerateExclusive()
454 TextIdx blockLen = static_cast<TextIdx>(block_len); in durchschlagGenerateExclusive()
460 const std::vector<TextIdx>& offsets = context.offsets; in durchschlagGenerateExclusive()
462 const TextIdx* shortcut = context.sliceMap.data(); in durchschlagGenerateExclusive()
473 TextIdx end = total - sliceLen + 1; in durchschlagGenerateExclusive()
475 TextIdx span = blockLen - sliceLen + 1; in durchschlagGenerateExclusive()
476 end = static_cast<TextIdx>(context.sliceMap.size()) - span; in durchschlagGenerateExclusive()
477 std::vector<TextIdx> candidates; in durchschlagGenerateExclusive()
478 std::vector<TextIdx> next(end); in durchschlagGenerateExclusive()
486 TextIdx mark = 0; in durchschlagGenerateExclusive()
489 TextIdx dictSize = calculateDictionarySize(ranges); in durchschlagGenerateExclusive()
498 TextIdx candidate = 0; in durchschlagGenerateExclusive()
537 for (TextIdx j = candidate; j < candidate + span; ++j) { in durchschlagGenerateExclusive()
553 TextIdx targetSize = static_cast<TextIdx>(dictionary_size_limit); in durchschlagGenerateCollaborative()
558 TextIdx sliceLen = context.sliceLen; in durchschlagGenerateCollaborative()
559 TextIdx total = context.dataSize; in durchschlagGenerateCollaborative()
560 TextIdx blockLen = static_cast<TextIdx>(block_len); in durchschlagGenerateCollaborative()
566 const std::vector<TextIdx>& offsets = context.offsets; in durchschlagGenerateCollaborative()
568 const TextIdx* shortcut = context.sliceMap.data(); in durchschlagGenerateCollaborative()
579 TextIdx end = total - sliceLen + 1; in durchschlagGenerateCollaborative()
581 TextIdx span = blockLen - sliceLen + 1; in durchschlagGenerateCollaborative()
582 end = static_cast<TextIdx>(context.sliceMap.size()) - span; in durchschlagGenerateCollaborative()
589 TextIdx mark = 0; in durchschlagGenerateCollaborative()
591 TextIdx dictSize = calculateDictionarySize(ranges); in durchschlagGenerateCollaborative()
603 TextIdx candidate = candidates[0].position; in durchschlagGenerateCollaborative()
613 for (TextIdx j = candidate; j < candidate + span; ++j) { in durchschlagGenerateCollaborative()
629 for (TextIdx j = candidate; j < candidate + span; ++j) { in durchschlagGenerateCollaborative()
663 const std::vector<TextIdx>& offsets = context.offsets; in durchschlag_distill()
665 const TextIdx* shortcut = context.sliceMap.data(); in durchschlag_distill()
666 TextIdx sliceLen = context.sliceLen; in durchschlag_distill()
667 TextIdx total = context.dataSize; in durchschlag_distill()
668 TextIdx end = total - sliceLen + 1; in durchschlag_distill()
672 TextIdx readPos = 0; in durchschlag_distill()
673 TextIdx writePos = 0; in durchschlag_distill()
674 TextIdx lastNonUniquePos = 0; in durchschlag_distill()
675 for (TextIdx i = 0; i < sample_sizes->size(); ++i) { in durchschlag_distill()
676 TextIdx sampleStart = writePos; in durchschlag_distill()
677 TextIdx oldSampleEnd = in durchschlag_distill()
678 readPos + static_cast<TextIdx>(sample_sizes->at(i)); in durchschlag_distill()
705 const std::vector<TextIdx>& offsets = context.offsets; in durchschlag_purify()
707 const TextIdx* shortcut = context.sliceMap.data(); in durchschlag_purify()
708 TextIdx sliceLen = context.sliceLen; in durchschlag_purify()
709 TextIdx total = context.dataSize; in durchschlag_purify()
710 TextIdx end = total - sliceLen + 1; in durchschlag_purify()
714 TextIdx lastNonUniquePos = 0; in durchschlag_purify()
715 for (TextIdx readPos = 0; readPos < total; ++readPos) { in durchschlag_purify()