Lines Matching refs:f
169 logger Logger, dbPath string) (f *Finder, err error) {
175 logger Logger, dbPath string, numThreads int) (f *Finder, err error) {
187 f = &Finder{
200 f.loadFromFilesystem()
203 err = f.getErr()
211 path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
213 node := f.nodes.GetNode(filepath.Clean(path), false)
219 return f, nil
223 func (f *Finder) FindAll() []string {
224 return f.FindAt("/")
228 func (f *Finder) FindAt(rootDir string) []string {
232 return f.FindMatching(rootDir, filter)
236 func (f *Finder) FindNamed(fileName string) []string {
237 return f.FindNamedAt("/", fileName)
243 func (f *Finder) FindNamedAt(rootPath string, fileName string) []string {
254 return f.FindMatching(rootPath, filter)
259 func (f *Finder) FindFirstNamed(fileName string) []string {
260 return f.FindFirstNamedAt("/", fileName)
265 func (f *Finder) FindFirstNamedAt(rootPath string, fileName string) []string {
279 return f.FindMatching(rootPath, filter)
286 func (f *Finder) FindMatching(rootPath string, filter WalkFunc) []string {
290 workingDir := f.cacheMetadata.Config.WorkingDirectory
300 f.verbosef("FindMatching waiting for finder to be idle\n")
301 f.lock()
302 defer f.unlock()
304 node := f.nodes.GetNode(rootPath, false)
306 f.verbosef("No data for path %v ; apparently not included in cache params: %v\n",
307 rootPath, f.cacheMetadata.Config.CacheParams)
313 f.verbosef("Finder finding %v using cache\n", rootPath)
314 results := f.findInCacheMultithreaded(node, filter, f.numSearchingThreads)
323 f.verbosef("Found %v files under %v in %v using cache\n",
330 func (f *Finder) Shutdown() {
331 f.WaitForDbDump()
335 func (f *Finder) WaitForDbDump() {
336 f.shutdownWaitgroup.Wait()
341 func (f *Finder) goDumpDb() {
342 if f.wasModified() {
343 f.shutdownWaitgroup.Add(1)
345 err := f.dumpDb()
347 f.verbosef("%v\n", err)
349 f.shutdownWaitgroup.Done()
352 f.verbosef("Skipping dumping unmodified db\n")
371 func (f *Finder) verbosef(format string, args ...interface{}) {
372 f.logger.Output(2, fmt.Sprintf(format, args...))
376 func (f *Finder) loadFromFilesystem() {
377 f.threadPool = newThreadPool(f.numDbLoadingThreads)
379 err := f.startFromExternalCache()
381 f.startWithoutExternalCache()
384 f.goDumpDb()
386 f.threadPool = nil
389 func (f *Finder) startFind(path string) {
391 path = filepath.Join(f.cacheMetadata.Config.WorkingDirectory, path)
393 node := f.nodes.GetNode(path, true)
394 f.statDirAsync(node)
397 func (f *Finder) lock() {
398 f.mutex.Lock()
401 func (f *Finder) unlock() {
402 f.mutex.Unlock()
651 func (f *Finder) serializeCacheEntry(dirInfos []dirFullInfo) ([]byte, error) {
700 func (f *Finder) parseCacheEntry(bytes []byte) ([]dirFullInfo, error) {
746 func (f *Finder) readLine(reader *bufio.Reader) ([]byte, error) {
751 func (f *Finder) validateCacheHeader(cacheReader *bufio.Reader) bool {
752 cacheVersionBytes, err := f.readLine(cacheReader)
754 f.verbosef("Failed to read database header; database is invalid\n")
761 currentVersion := f.cacheMetadata.Version
763 …f.verbosef("Version changed from %q to %q, database is not applicable\n", cacheVersionString, curr…
767 cacheParamBytes, err := f.readLine(cacheReader)
769 f.verbosef("Failed to read database search params; database is invalid\n")
777 currentParamBytes, err := f.cacheMetadata.Config.Dump()
784 …f.verbosef("Params changed from %q to %q, database is not applicable\n", cacheParamString, current…
792 func (f *Finder) loadBytes(id int, data []byte) (m *pathMap, dirsToWalk []string, err error) {
796 cachedNodes, err := f.parseCacheEntry(data)
802 f.verbosef("Unmarshaled %v objects for %v in %v\n",
810 stats[i] = f.statDirSync(node.Path)
822 if !f.isInfoUpToDate(cachedNode.statResponse, updated) && updated.ModTime != 0 {
823 f.setModified()
834 f.verbosef("Statted inodes of block %v in %v\n", id, time.Now().Sub(unmarshalDate))
841 func (f *Finder) startFromExternalCache() (err error) {
843 dbPath := f.DbPath
846 reader, err := f.filesystem.Open(dbPath)
851 if !f.validateCacheHeader(bufferedReader) {
854 f.verbosef("Database header matches, will attempt to use database %v\n", f.DbPath)
866 blockChannel := make(chan dataBlock, f.numDbLoadingThreads)
874 data, err := f.readLine(bufferedReader)
887 f.verbosef("Read block %v after %v\n", index, duration)
889 f.verbosef("Read %v blocks in %v\n", index, duration)
907 threadPool := newThreadPool(f.numDbLoadingThreads)
924 f.verbosef("Starting to process block %v after %v\n",
926 tempMap, updatedDirs, err := f.loadBytes(block.id, block.data)
929 f.verbosef(
941 f.verbosef("Processed block %v in %v\n",
949 f.verbosef("Finished processing %v blocks in %v\n",
991 f.nodes = *mainTree
997 f.listDirsAsync(nodesToWalk[i])
999 f.verbosef("Loaded db and statted known dirs in %v\n", time.Since(startTime))
1000 f.threadPool.Wait()
1001 f.verbosef("Loaded db and statted all dirs in %v\n", time.Now().Sub(startTime))
1008 func (f *Finder) startWithoutExternalCache() {
1010 configDirs := f.cacheMetadata.Config.RootDirs
1034 f.verbosef("Starting find of %v\n", path)
1035 f.startFind(path)
1038 f.threadPool.Wait()
1040 f.verbosef("Scanned filesystem (not using cache) in %v\n", time.Now().Sub(startTime))
1044 func (f *Finder) isInfoUpToDate(old statResponse, new statResponse) (equal bool) {
1057 func (f *Finder) wasModified() bool {
1058 return atomic.LoadInt32(&f.modifiedFlag) > 0
1061 func (f *Finder) setModified() {
1064 atomic.StoreInt32(&f.modifiedFlag, newVal)
1068 func (f *Finder) sortedDirEntries() []dirFullInfo {
1071 for _, node := range f.nodes.DumpAll() {
1077 f.verbosef("Generated %v cache entries in %v\n", len(nodes), discoveryDate.Sub(startTime))
1083 f.verbosef("Sorted %v cache entries in %v\n", len(nodes), sortDate.Sub(discoveryDate))
1089 func (f *Finder) serializeDb() ([]byte, error) {
1091 var entryList = f.sortedDirEntries()
1099 header = append(header, []byte(f.cacheMetadata.Version)...)
1101 configDump, err := f.cacheMetadata.Config.Dump()
1108 numBlocks := f.numDbLoadingThreads
1126 byteBlock, subErr := f.serializeCacheEntry(block)
1127 f.verbosef("Serialized block %v into %v bytes\n", index, len(byteBlock))
1129 f.verbosef("%v\n", subErr.Error())
1154 func (f *Finder) dumpDb() error {
1156 f.verbosef("Dumping db\n")
1158 tempPath := f.DbPath + ".tmp"
1160 bytes, err := f.serializeDb()
1165 f.verbosef("Serialized db in %v\n", serializeDate.Sub(startTime))
1167 err = f.filesystem.WriteFile(tempPath, bytes, 0777)
1171 err = f.filesystem.Rename(tempPath, f.DbPath)
1176 f.verbosef("Wrote db in %v\n", time.Now().Sub(serializeDate))
1182 func (f *Finder) canIgnoreFsErr(err error) bool {
1203 func (f *Finder) onFsError(path string, err error) {
1204 if !f.canIgnoreFsErr(err) {
1210 f.errlock.Lock()
1211 f.fsErrs = append(f.fsErrs, fsErr{path: path, err: err})
1212 f.errlock.Unlock()
1217 func (f *Finder) discardErrsForPrunedPaths() {
1220 relevantErrs := make([]fsErr, 0, len(f.fsErrs))
1221 for _, fsErr := range f.fsErrs {
1223 node := f.nodes.GetNode(path, false)
1230 f.fsErrs = relevantErrs
1234 func (f *Finder) getErr() error {
1235 f.discardErrsForPrunedPaths()
1237 numErrs := len(f.fsErrs)
1245 …message = fmt.Sprintf("finder encountered %v errors: %v...", numErrs, f.fsErrs[:maxNumErrsToInclud…
1247 message = fmt.Sprintf("finder encountered %v errors: %v", numErrs, f.fsErrs)
1253 func (f *Finder) statDirAsync(dir *pathMap) {
1256 f.threadPool.Run(
1258 updatedStats := f.statDirSync(path)
1260 if !f.isInfoUpToDate(node.statResponse, updatedStats) {
1265 f.setModified()
1269 f.listDirAsync(dir)
1276 func (f *Finder) statDirSync(path string) statResponse {
1278 fileInfo, err := f.filesystem.Lstat(path)
1283 f.onFsError(path, err)
1289 inode, err := f.filesystem.InodeNumber(fileInfo)
1294 device, err := f.filesystem.DeviceNumber(fileInfo)
1299 permissionsChangeTime, err := f.filesystem.PermTime(fileInfo)
1316 func (f *Finder) shouldIncludeFile(fileName string) bool {
1317 for _, includedName := range f.cacheMetadata.Config.IncludeFiles {
1322 for _, includeSuffix := range f.cacheMetadata.Config.IncludeSuffixes {
1331 func (f *Finder) pruneCacheCandidates(items *DirEntries) {
1334 for _, abortedName := range f.cacheMetadata.Config.PruneFiles {
1346 if f.shouldIncludeFile(fileName) {
1359 for _, excludedName := range f.cacheMetadata.Config.ExcludeDirs {
1374 func (f *Finder) listDirsAsync(nodes []*pathMap) {
1375 f.threadPool.Run(
1378 f.listDirSync(nodes[i])
1384 func (f *Finder) listDirAsync(node *pathMap) {
1385 f.threadPool.Run(
1387 f.listDirSync(node)
1392 func (f *Finder) listDirSync(dir *pathMap) {
1394 children, err := f.filesystem.ReadDir(path)
1398 f.onFsError(path, err)
1412 childStat, err := f.filesystem.Stat(childPath)
1436 f.pruneCacheCandidates(entry)
1446 f.statDirAsync(childNode)
1466 func (f *Finder) listMatches(node *pathMap,
1495 func (f *Finder) findInCacheMultithreaded(node *pathMap, filter WalkFunc,
1500 return f.findInCacheSinglethreaded(node, filter)
1509 subDirs, filePaths := f.listMatches(node, filter)
1515 childResults := f.findInCacheMultithreaded(child, filter, numChildThreads)
1536 func (f *Finder) findInCacheSinglethreaded(node *pathMap, filter WalkFunc) []string {
1548 subDirs, filePaths := f.listMatches(currentNode, filter)