1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_
17 #define TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_
18 
19 #include <string>
20 #include <unordered_map>
21 #include <vector>
22 
23 #include <curl/curl.h>
24 #include "tensorflow/core/platform/cloud/http_request.h"
25 #include "tensorflow/core/platform/env.h"
26 #include "tensorflow/core/platform/errors.h"
27 #include "tensorflow/core/platform/macros.h"
28 #include "tensorflow/core/platform/protobuf.h"
29 #include "tensorflow/core/platform/status.h"
30 #include "tensorflow/core/platform/stringpiece.h"
31 #include "tensorflow/core/platform/types.h"
32 
33 namespace tensorflow {
34 
35 class LibCurl;  // libcurl interface as a class, for dependency injection.
36 
37 /// \brief A basic HTTP client based on the libcurl library.
38 ///
39 /// The usage pattern for the class reflects the one of the libcurl library:
40 /// create a request object, set request parameters and call Send().
41 ///
42 /// For example:
43 ///   std::unique_ptr<HttpRequest> request(http_request_factory->Create());
44 ///   request->SetUri("http://www.google.com");
45 ///   request->SetResultsBuffer(out_buffer);
46 ///   request->Send();
47 class CurlHttpRequest : public HttpRequest {
48  public:
49   class Factory : public HttpRequest::Factory {
50    public:
~Factory()51     virtual ~Factory() {}
Create()52     virtual HttpRequest* Create() { return new CurlHttpRequest(); }
53   };
54 
55   CurlHttpRequest();
CurlHttpRequest(LibCurl * libcurl)56   explicit CurlHttpRequest(LibCurl* libcurl)
57       : CurlHttpRequest(libcurl, Env::Default()) {}
58   CurlHttpRequest(LibCurl* libcurl, Env* env);
59   ~CurlHttpRequest() override;
60 
61   /// Sets the request URI.
62   void SetUri(const string& uri) override;
63 
64   /// \brief Sets the Range header.
65   ///
66   /// Used for random seeks, for example "0-999" returns the first 1000 bytes
67   /// (note that the right border is included).
68   void SetRange(uint64 start, uint64 end) override;
69 
70   /// Sets a request header.
71   void AddHeader(const string& name, const string& value) override;
72 
73   void AddResolveOverride(const string& hostname, int64 port,
74                           const string& ip_addr) override;
75 
76   /// Sets the 'Authorization' header to the value of 'Bearer ' + auth_token.
77   void AddAuthBearerHeader(const string& auth_token) override;
78 
79   void SetRequestStats(RequestStats* stats) override;
80 
81   /// Makes the request a DELETE request.
82   void SetDeleteRequest() override;
83 
84   /// \brief Makes the request a PUT request.
85   ///
86   /// The request body will be taken from the specified file starting from
87   /// the given offset.
88   Status SetPutFromFile(const string& body_filepath, size_t offset) override;
89 
90   /// Makes the request a PUT request with an empty body.
91   void SetPutEmptyBody() override;
92 
93   /// \brief Makes the request a POST request.
94   ///
95   /// The request body will be taken from the specified buffer.
96   void SetPostFromBuffer(const char* buffer, size_t size) override;
97 
98   /// Makes the request a POST request with an empty body.
99   void SetPostEmptyBody() override;
100 
101   /// \brief Specifies the buffer for receiving the response body.
102   ///
103   /// Size of out_buffer after an access will be exactly the number of bytes
104   /// read. Existing content of the vector will be cleared.
105   void SetResultBuffer(std::vector<char>* out_buffer) override;
106 
107   /// \brief Specifies the buffer for receiving the response body, when the
108   /// caller knows the maximum size of the response body.
109   ///
110   /// This method allows the caller to receive the response body without an
111   /// additional intermediate buffer allocation and copy.  This method should
112   /// be called before calling Send(). After Send() has succeeded, the caller
113   /// should use the GetResultBufferDirectBytesTransferred() method in order
114   /// to learn how many bytes were transferred.
115   ///
116   /// Using this method is mutually exclusive with using SetResultBuffer().
117   void SetResultBufferDirect(char* buffer, size_t size) override;
118 
119   /// \brief Distinguish response type (direct vs. implicit).
120   bool IsDirectResponse() const;
121 
122   /// \brief Returns the number of bytes (of the response body) that were
123   /// transferred, when using the SetResultBufferDirect() method. The returned
124   /// value will always be less than or equal to the 'size' parameter that
125   /// was passed to SetResultBufferDirect(). If the actual HTTP response body
126   /// was greater than 'size' bytes, then this transfer method will only copy
127   /// the first 'size' bytes, and the rest will be ignored.
128   size_t GetResultBufferDirectBytesTransferred() override;
129 
130   /// \brief Returns the response headers of a completed request.
131   ///
132   /// If the header is not found, returns an empty string.
133   string GetResponseHeader(const string& name) const override;
134 
135   /// Returns the response code of a completed request.
136   uint64 GetResponseCode() const override;
137 
138   /// \brief Sends the formed request.
139   ///
140   /// If the result buffer was defined, the response will be written there.
141   /// The object is not designed to be re-used after Send() is executed.
142   Status Send() override;
143 
144   // Url encodes str and returns a new string.
145   string EscapeString(const string& str) override;
146 
147   void SetTimeouts(uint32 connection, uint32 inactivity, uint32 total) override;
148 
149  private:
150   /// A write callback in the form which can be accepted by libcurl.
151   static size_t WriteCallback(const void* ptr, size_t size, size_t nmemb,
152                               void* userdata);
153 
154   /// Processes response body content received when using SetResultBufferDirect.
155   static size_t WriteCallbackDirect(const void* ptr, size_t size, size_t nmemb,
156                                     void* userdata);
157   /// A read callback in the form which can be accepted by libcurl.
158   static size_t ReadCallback(void* ptr, size_t size, size_t nmemb,
159                              FILE* userdata);
160   /// A header callback in the form which can be accepted by libcurl.
161   static size_t HeaderCallback(const void* ptr, size_t size, size_t nmemb,
162                                void* this_object);
163   /// A progress meter callback in the form which can be accepted by libcurl.
164   static int ProgressCallback(void* this_object, curl_off_t dltotal,
165                               curl_off_t dlnow, curl_off_t ultotal,
166                               curl_off_t ulnow);
167   void CheckMethodNotSet() const;
168   void CheckNotSent() const;
169   StringPiece GetResponse() const;
170 
171   /// Helper to convert the given CURLcode and error buffer, representing the
172   /// result of performing a transfer, into a Status with an error message.
173   Status CURLcodeToStatus(CURLcode code, const char* error_buffer);
174 
175   LibCurl* libcurl_;
176   Env* env_;
177 
178   FILE* put_body_ = nullptr;
179 
180   StringPiece post_body_buffer_;
181   size_t post_body_read_ = 0;
182 
183   std::vector<char>* response_buffer_ = nullptr;
184 
185   struct DirectResponseState {
186     char* buffer_;
187     size_t buffer_size_;
188     size_t bytes_transferred_;
189     size_t bytes_received_;
190   };
191   DirectResponseState direct_response_ = {};
192 
193   CURL* curl_ = nullptr;
194   curl_slist* curl_headers_ = nullptr;
195   curl_slist* resolve_list_ = nullptr;
196 
197   RequestStats* stats_ = nullptr;
198 
199   std::vector<char> default_response_buffer_;
200 
201   std::unordered_map<string, string> response_headers_;
202   uint64 response_code_ = 0;
203 
204   // The timestamp of the last activity related to the request execution, in
205   // seconds since epoch.
206   uint64 last_progress_timestamp_ = 0;
207   // The last progress in terms of bytes transmitted.
208   curl_off_t last_progress_bytes_ = 0;
209 
210   // The maximum period of request inactivity.
211   uint32 inactivity_timeout_secs_ = 60;  // 1 minute
212 
213   // Timeout for the connection phase.
214   uint32 connect_timeout_secs_ = 120;  // 2 minutes
215 
216   // Timeout for the whole request. Set only to prevent hanging indefinitely.
217   uint32 request_timeout_secs_ = 3600;  // 1 hour
218 
219   // Members to enforce the usage flow.
220   bool is_uri_set_ = false;
221   bool is_method_set_ = false;
222   bool is_sent_ = false;
223 
224   // Store the URI to help disambiguate requests when errors occur.
225   string uri_;
226   RequestMethod method_ = RequestMethod::kGet;
227 
228   // Limit the size of an http response that is copied into an error message.
229   const size_t response_to_error_limit_ = 500;
230 
231   TF_DISALLOW_COPY_AND_ASSIGN(CurlHttpRequest);
232 };
233 
234 /// \brief A proxy to the libcurl C interface as a dependency injection measure.
235 ///
236 /// This class is meant as a very thin wrapper for the libcurl C library.
237 class LibCurl {
238  public:
~LibCurl()239   virtual ~LibCurl() {}
240 
241   virtual CURL* curl_easy_init() = 0;
242   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
243                                     uint64 param) TF_MUST_USE_RESULT = 0;
244   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
245                                     const char* param) TF_MUST_USE_RESULT = 0;
246   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
247                                     void* param) TF_MUST_USE_RESULT = 0;
248   virtual CURLcode curl_easy_setopt(
249       CURL* curl, CURLoption option,
250       size_t (*param)(void*, size_t, size_t, FILE*)) TF_MUST_USE_RESULT = 0;
251   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
252                                     size_t (*param)(const void*, size_t, size_t,
253                                                     void*))
254       TF_MUST_USE_RESULT = 0;
255   virtual CURLcode curl_easy_setopt(
256       CURL* curl, CURLoption option,
257       int (*param)(void* clientp, curl_off_t dltotal, curl_off_t dlnow,
258                    curl_off_t ultotal,
259                    curl_off_t ulnow)) TF_MUST_USE_RESULT = 0;
260   virtual CURLcode curl_easy_perform(CURL* curl) TF_MUST_USE_RESULT = 0;
261   virtual CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
262                                      uint64* value) TF_MUST_USE_RESULT = 0;
263   virtual CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
264                                      double* value) TF_MUST_USE_RESULT = 0;
265   virtual void curl_easy_cleanup(CURL* curl) = 0;
266   virtual curl_slist* curl_slist_append(curl_slist* list, const char* str) = 0;
267   virtual void curl_slist_free_all(curl_slist* list) = 0;
268   virtual char* curl_easy_escape(CURL* curl, const char* str, int length) = 0;
269   virtual void curl_free(void* p) = 0;
270 };
271 
272 }  // namespace tensorflow
273 
274 #endif  // TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_
275