1 /* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/core/platform/file_system.h"
17 
18 #include <sys/stat.h>
19 
20 #include "tensorflow/core/lib/core/status_test_util.h"
21 #include "tensorflow/core/lib/io/path.h"
22 #include "tensorflow/core/lib/strings/str_util.h"
23 #include "tensorflow/core/lib/strings/strcat.h"
24 #include "tensorflow/core/platform/null_file_system.h"
25 #include "tensorflow/core/platform/test.h"
26 
27 namespace tensorflow {
28 
29 static const char* const kPrefix = "ipfs://solarsystem";
30 
31 // A file system that has Planets, Satellites and Sub Satellites. Sub satellites
32 // cannot have children further.
33 class InterPlanetaryFileSystem : public NullFileSystem {
34  public:
FileExists(const string & fname)35   Status FileExists(const string& fname) override {
36     string parsed_path;
37     ParsePath(fname, &parsed_path);
38     if (BodyExists(parsed_path)) {
39       return Status::OK();
40     }
41     return Status(tensorflow::error::NOT_FOUND, "File does not exist");
42   }
43 
44   // Adds the dir to the parent's children list and creates an entry for itself.
CreateDir(const string & dirname)45   Status CreateDir(const string& dirname) override {
46     string parsed_path;
47     ParsePath(dirname, &parsed_path);
48     // If the directory already exists, throw an error.
49     if (celestial_bodies_.find(parsed_path) != celestial_bodies_.end()) {
50       return Status(tensorflow::error::ALREADY_EXISTS,
51                     "dirname already exists.");
52     }
53     std::vector<string> split_path = str_util::Split(parsed_path, '/');
54     // If the path is too long then we don't support it.
55     if (split_path.size() > 3) {
56       return Status(tensorflow::error::INVALID_ARGUMENT, "Bad dirname");
57     }
58     if (split_path.empty()) {
59       return Status::OK();
60     }
61     if (split_path.size() == 1) {
62       celestial_bodies_[""].insert(parsed_path);
63       celestial_bodies_.insert(
64           std::pair<string, std::set<string>>(parsed_path, {}));
65       return Status::OK();
66     }
67     if (split_path.size() == 2) {
68       if (!BodyExists(split_path[0])) {
69         return Status(tensorflow::error::FAILED_PRECONDITION,
70                       "Base dir not created");
71       }
72       celestial_bodies_[split_path[0]].insert(split_path[1]);
73       celestial_bodies_.insert(
74           std::pair<string, std::set<string>>(parsed_path, {}));
75       return Status::OK();
76     }
77     if (split_path.size() == 3) {
78       const string& parent_path = io::JoinPath(split_path[0], split_path[1]);
79       if (!BodyExists(parent_path)) {
80         return Status(tensorflow::error::FAILED_PRECONDITION,
81                       "Base dir not created");
82       }
83       celestial_bodies_[parent_path].insert(split_path[2]);
84       celestial_bodies_.insert(
85           std::pair<string, std::set<string>>(parsed_path, {}));
86       return Status::OK();
87     }
88     return Status(tensorflow::error::FAILED_PRECONDITION, "Failed to create");
89   }
90 
IsDirectory(const string & dirname)91   Status IsDirectory(const string& dirname) override {
92     string parsed_path;
93     ParsePath(dirname, &parsed_path);
94     // Simulate evil_directory has bad permissions by throwing a LOG(FATAL)
95     if (parsed_path == "evil_directory") {
96       LOG(FATAL) << "evil_directory cannot be accessed";
97     }
98     std::vector<string> split_path = str_util::Split(parsed_path, '/');
99     if (split_path.size() > 2) {
100       return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir");
101     }
102     if (celestial_bodies_.find(parsed_path) != celestial_bodies_.end()) {
103       return Status::OK();
104     }
105     return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir");
106   }
107 
GetChildren(const string & dir,std::vector<string> * result)108   Status GetChildren(const string& dir, std::vector<string>* result) override {
109     TF_RETURN_IF_ERROR(IsDirectory(dir));
110     string parsed_path;
111     ParsePath(dir, &parsed_path);
112     result->insert(result->begin(), celestial_bodies_[parsed_path].begin(),
113                    celestial_bodies_[parsed_path].end());
114     return Status::OK();
115   }
116 
117  private:
BodyExists(const string & name)118   bool BodyExists(const string& name) {
119     return celestial_bodies_.find(name) != celestial_bodies_.end();
120   }
121 
ParsePath(const string & name,string * parsed_path)122   void ParsePath(const string& name, string* parsed_path) {
123     StringPiece scheme, host, path;
124     io::ParseURI(name, &scheme, &host, &path);
125     ASSERT_EQ(scheme, "ipfs");
126     ASSERT_EQ(host, "solarsystem");
127     str_util::ConsumePrefix(&path, "/");
128     *parsed_path = string(path);
129   }
130 
131   std::map<string, std::set<string>> celestial_bodies_ = {
132       std::pair<string, std::set<string>>(
133           "", {"Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn",
134                "Uranus", "Neptune"}),
135       std::pair<string, std::set<string>>("Mercury", {}),
136       std::pair<string, std::set<string>>("Venus", {}),
137       std::pair<string, std::set<string>>("Earth", {"Moon"}),
138       std::pair<string, std::set<string>>("Mars", {}),
139       std::pair<string, std::set<string>>("Jupiter",
140                                           {"Europa", "Io", "Ganymede"}),
141       std::pair<string, std::set<string>>("Saturn", {}),
142       std::pair<string, std::set<string>>("Uranus", {}),
143       std::pair<string, std::set<string>>("Neptune", {}),
144       std::pair<string, std::set<string>>("Earth/Moon", {}),
145       std::pair<string, std::set<string>>("Jupiter/Europa", {}),
146       std::pair<string, std::set<string>>("Jupiter/Io", {}),
147       std::pair<string, std::set<string>>("Jupiter/Ganymede", {})};
148 };
149 
150 // Returns all the matched entries as a comma separated string removing the
151 // common prefix of BaseDir().
Match(InterPlanetaryFileSystem * ipfs,const string & suffix_pattern)152 string Match(InterPlanetaryFileSystem* ipfs, const string& suffix_pattern) {
153   std::vector<string> results;
154   Status s =
155       ipfs->GetMatchingPaths(io::JoinPath(kPrefix, suffix_pattern), &results);
156   if (!s.ok()) {
157     return s.ToString();
158   } else {
159     std::vector<StringPiece> trimmed_results;
160     std::sort(results.begin(), results.end());
161     for (const string& result : results) {
162       StringPiece trimmed_result(result);
163       EXPECT_TRUE(str_util::ConsumePrefix(&trimmed_result,
164                                           strings::StrCat(kPrefix, "/")));
165       trimmed_results.push_back(trimmed_result);
166     }
167     return str_util::Join(trimmed_results, ",");
168   }
169 }
170 
TEST(InterPlanetaryFileSystemTest,IPFSMatch)171 TEST(InterPlanetaryFileSystemTest, IPFSMatch) {
172   InterPlanetaryFileSystem ipfs;
173   EXPECT_EQ(Match(&ipfs, "thereisnosuchfile"), "");
174   EXPECT_EQ(Match(&ipfs, "*"),
175             "Earth,Jupiter,Mars,Mercury,Neptune,Saturn,Uranus,Venus");
176   // Returns Jupiter's moons.
177   EXPECT_EQ(Match(&ipfs, "Jupiter/*"),
178             "Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io");
179   // Returns Jupiter's and Earth's moons.
180   EXPECT_EQ(Match(&ipfs, "*/*"),
181             "Earth/Moon,Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io");
182   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "Planet0")));
183   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "Planet1")));
184   EXPECT_EQ(Match(&ipfs, "Planet[0-1]"), "Planet0,Planet1");
185   EXPECT_EQ(Match(&ipfs, "Planet?"), "Planet0,Planet1");
186 }
187 
TEST(InterPlanetaryFileSystemTest,MatchSimple)188 TEST(InterPlanetaryFileSystemTest, MatchSimple) {
189   InterPlanetaryFileSystem ipfs;
190   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-00")));
191   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-0a")));
192   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-01")));
193   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "match-aaa")));
194 
195   EXPECT_EQ(Match(&ipfs, "match-*"), "match-00,match-01,match-0a,match-aaa");
196   EXPECT_EQ(Match(&ipfs, "match-0[0-9]"), "match-00,match-01");
197   EXPECT_EQ(Match(&ipfs, "match-?[0-9]"), "match-00,match-01");
198   EXPECT_EQ(Match(&ipfs, "match-?a*"), "match-0a,match-aaa");
199   EXPECT_EQ(Match(&ipfs, "match-??"), "match-00,match-01,match-0a");
200 }
201 
202 // Create 2 directories abcd and evil_directory. Look for abcd and make sure
203 // that evil_directory isn't accessed.
TEST(InterPlanetaryFileSystemTest,MatchOnlyNeeded)204 TEST(InterPlanetaryFileSystemTest, MatchOnlyNeeded) {
205   InterPlanetaryFileSystem ipfs;
206   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "abcd")));
207   TF_EXPECT_OK(ipfs.CreateDir(io::JoinPath(kPrefix, "evil_directory")));
208 
209   EXPECT_EQ(Match(&ipfs, "abcd"), "abcd");
210 }
211 
TEST(InterPlanetaryFileSystemTest,MatchDirectory)212 TEST(InterPlanetaryFileSystemTest, MatchDirectory) {
213   InterPlanetaryFileSystem ipfs;
214   TF_EXPECT_OK(
215       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/x")));
216   TF_EXPECT_OK(
217       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-0a/abc/x")));
218   TF_EXPECT_OK(
219       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/x")));
220   TF_EXPECT_OK(
221       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-aaa/abc/x")));
222 
223   EXPECT_EQ(Match(&ipfs, "match-*/abc/x"),
224             "match-00/abc/x,match-01/abc/x,match-0a/abc/x,match-aaa/abc/x");
225   EXPECT_EQ(Match(&ipfs, "match-0[0-9]/abc/x"),
226             "match-00/abc/x,match-01/abc/x");
227   EXPECT_EQ(Match(&ipfs, "match-?[0-9]/abc/x"),
228             "match-00/abc/x,match-01/abc/x");
229   EXPECT_EQ(Match(&ipfs, "match-?a*/abc/x"), "match-0a/abc/x,match-aaa/abc/x");
230   EXPECT_EQ(Match(&ipfs, "match-?[^a]/abc/x"), "match-00/abc/x,match-01/abc/x");
231 }
232 
TEST(InterPlanetaryFileSystemTest,MatchMultipleWildcards)233 TEST(InterPlanetaryFileSystemTest, MatchMultipleWildcards) {
234   InterPlanetaryFileSystem ipfs;
235   TF_EXPECT_OK(
236       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/00")));
237   TF_EXPECT_OK(
238       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/01")));
239   TF_EXPECT_OK(
240       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-00/abc/09")));
241   TF_EXPECT_OK(
242       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/00")));
243   TF_EXPECT_OK(
244       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/04")));
245   TF_EXPECT_OK(
246       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-01/abc/10")));
247   TF_EXPECT_OK(
248       ipfs.RecursivelyCreateDir(io::JoinPath(kPrefix, "match-02/abc/00")));
249 
250   EXPECT_EQ(Match(&ipfs, "match-0[0-1]/abc/0[0-8]"),
251             "match-00/abc/00,match-00/abc/01,match-01/abc/00,match-01/abc/04");
252 }
253 
TEST(InterPlanetaryFileSystemTest,RecursivelyCreateAlreadyExistingDir)254 TEST(InterPlanetaryFileSystemTest, RecursivelyCreateAlreadyExistingDir) {
255   InterPlanetaryFileSystem ipfs;
256   const string dirname = io::JoinPath(kPrefix, "match-00/abc/00");
257   TF_EXPECT_OK(ipfs.RecursivelyCreateDir(dirname));
258   // Ensure that CreateDir throws an error, to sanity check that this test
259   // actually tests the behavior of RecursivelyCreateDir.
260   EXPECT_EQ(ipfs.CreateDir(dirname).code(), tensorflow::error::ALREADY_EXISTS);
261   TF_EXPECT_OK(ipfs.RecursivelyCreateDir(dirname));
262 }
263 
264 // A simple file system with a root directory and a single file underneath it.
265 class TestFileSystem : public NullFileSystem {
266  public:
267   // Only allow for a single root directory.
IsDirectory(const string & dirname)268   Status IsDirectory(const string& dirname) override {
269     if (dirname == "." || dirname.empty()) {
270       return Status::OK();
271     }
272     return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir");
273   }
274 
275   // Simulating a FS with a root dir and a single file underneath it.
GetChildren(const string & dir,std::vector<string> * result)276   Status GetChildren(const string& dir, std::vector<string>* result) override {
277     if (dir == "." || dir.empty()) {
278       result->push_back("test");
279     }
280     return Status::OK();
281   }
282 };
283 
284 // Making sure that ./<pattern> and <pattern> have the same result.
TEST(TestFileSystemTest,RootDirectory)285 TEST(TestFileSystemTest, RootDirectory) {
286   TestFileSystem fs;
287   std::vector<string> results;
288   auto ret = fs.GetMatchingPaths("./te*", &results);
289   EXPECT_EQ(1, results.size());
290   EXPECT_EQ("./test", results[0]);
291   ret = fs.GetMatchingPaths("te*", &results);
292   EXPECT_EQ(1, results.size());
293   EXPECT_EQ("./test", results[0]);
294 }
295 
296 }  // namespace tensorflow
297