1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_
17 #define TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_
18 
19 #include <string>
20 #include <unordered_map>
21 #include <vector>
22 #include <curl/curl.h>
23 #include "tensorflow/core/lib/core/errors.h"
24 #include "tensorflow/core/lib/core/status.h"
25 #include "tensorflow/core/lib/core/stringpiece.h"
26 #include "tensorflow/core/platform/cloud/http_request.h"
27 #include "tensorflow/core/platform/env.h"
28 #include "tensorflow/core/platform/macros.h"
29 #include "tensorflow/core/platform/protobuf.h"
30 #include "tensorflow/core/platform/types.h"
31 
32 namespace tensorflow {
33 
34 class LibCurl;  // libcurl interface as a class, for dependency injection.
35 
36 /// \brief A basic HTTP client based on the libcurl library.
37 ///
38 /// The usage pattern for the class reflects the one of the libcurl library:
39 /// create a request object, set request parameters and call Send().
40 ///
41 /// For example:
42 ///   std::unique_ptr<HttpRequest> request(http_request_factory->Create());
43 ///   request->SetUri("http://www.google.com");
44 ///   request->SetResultsBuffer(out_buffer);
45 ///   request->Send();
46 class CurlHttpRequest : public HttpRequest {
47  public:
48   class Factory : public HttpRequest::Factory {
49    public:
~Factory()50     virtual ~Factory() {}
Create()51     virtual HttpRequest* Create() { return new CurlHttpRequest(); }
52   };
53 
54   CurlHttpRequest();
CurlHttpRequest(LibCurl * libcurl)55   explicit CurlHttpRequest(LibCurl* libcurl)
56       : CurlHttpRequest(libcurl, Env::Default()) {}
57   CurlHttpRequest(LibCurl* libcurl, Env* env);
58   ~CurlHttpRequest() override;
59 
60   /// Sets the request URI.
61   void SetUri(const string& uri) override;
62 
63   /// \brief Sets the Range header.
64   ///
65   /// Used for random seeks, for example "0-999" returns the first 1000 bytes
66   /// (note that the right border is included).
67   void SetRange(uint64 start, uint64 end) override;
68 
69   /// Sets a request header.
70   void AddHeader(const string& name, const string& value) override;
71 
72   void AddResolveOverride(const string& hostname, int64 port,
73                           const string& ip_addr) override;
74 
75   /// Sets the 'Authorization' header to the value of 'Bearer ' + auth_token.
76   void AddAuthBearerHeader(const string& auth_token) override;
77 
78   void SetRequestStats(RequestStats* stats) override;
79 
80   /// Makes the request a DELETE request.
81   void SetDeleteRequest() override;
82 
83   /// \brief Makes the request a PUT request.
84   ///
85   /// The request body will be taken from the specified file starting from
86   /// the given offset.
87   Status SetPutFromFile(const string& body_filepath, size_t offset) override;
88 
89   /// Makes the request a PUT request with an empty body.
90   void SetPutEmptyBody() override;
91 
92   /// \brief Makes the request a POST request.
93   ///
94   /// The request body will be taken from the specified buffer.
95   void SetPostFromBuffer(const char* buffer, size_t size) override;
96 
97   /// Makes the request a POST request with an empty body.
98   void SetPostEmptyBody() override;
99 
100   /// \brief Specifies the buffer for receiving the response body.
101   ///
102   /// Size of out_buffer after an access will be exactly the number of bytes
103   /// read. Existing content of the vector will be cleared.
104   void SetResultBuffer(std::vector<char>* out_buffer) override;
105 
106   /// \brief Specifies the buffer for receiving the response body, when the
107   /// caller knows the maximum size of the response body.
108   ///
109   /// This method allows the caller to receive the response body without an
110   /// additional intermediate buffer allocation and copy.  This method should
111   /// be called before calling Send(). After Send() has succeeded, the caller
112   /// should use the GetResultBufferDirectBytesTransferred() method in order
113   /// to learn how many bytes were transferred.
114   ///
115   /// Using this method is mutually exclusive with using SetResultBuffer().
116   void SetResultBufferDirect(char* buffer, size_t size) override;
117 
118   /// \brief Distinguish response type (direct vs. implicit).
119   bool IsDirectResponse() const;
120 
121   /// \brief Returns the number of bytes (of the response body) that were
122   /// transferred, when using the SetResultBufferDirect() method. The returned
123   /// value will always be less than or equal to the 'size' parameter that
124   /// was passed to SetResultBufferDirect(). If the actual HTTP response body
125   /// was greater than 'size' bytes, then this transfer method will only copy
126   /// the first 'size' bytes, and the rest will be ignored.
127   size_t GetResultBufferDirectBytesTransferred() override;
128 
129   /// \brief Returns the response headers of a completed request.
130   ///
131   /// If the header is not found, returns an empty string.
132   string GetResponseHeader(const string& name) const override;
133 
134   /// Returns the response code of a completed request.
135   uint64 GetResponseCode() const override;
136 
137   /// \brief Sends the formed request.
138   ///
139   /// If the result buffer was defined, the response will be written there.
140   /// The object is not designed to be re-used after Send() is executed.
141   Status Send() override;
142 
143   // Url encodes str and returns a new string.
144   string EscapeString(const string& str) override;
145 
146   void SetTimeouts(uint32 connection, uint32 inactivity, uint32 total) override;
147 
148  private:
149   /// A write callback in the form which can be accepted by libcurl.
150   static size_t WriteCallback(const void* ptr, size_t size, size_t nmemb,
151                               void* userdata);
152 
153   /// Processes response body content received when using SetResultBufferDirect.
154   static size_t WriteCallbackDirect(const void* ptr, size_t size, size_t nmemb,
155                                     void* userdata);
156   /// A read callback in the form which can be accepted by libcurl.
157   static size_t ReadCallback(void* ptr, size_t size, size_t nmemb,
158                              FILE* userdata);
159   /// A header callback in the form which can be accepted by libcurl.
160   static size_t HeaderCallback(const void* ptr, size_t size, size_t nmemb,
161                                void* this_object);
162   /// A progress meter callback in the form which can be accepted by libcurl.
163   static int ProgressCallback(void* this_object, curl_off_t dltotal,
164                               curl_off_t dlnow, curl_off_t ultotal,
165                               curl_off_t ulnow);
166   void CheckMethodNotSet() const;
167   void CheckNotSent() const;
168   StringPiece GetResponse() const;
169 
170   /// Helper to convert the given CURLcode and error buffer, representing the
171   /// result of performing a transfer, into a Status with an error message.
172   Status CURLcodeToStatus(CURLcode code, const char* error_buffer);
173 
174   LibCurl* libcurl_;
175   Env* env_;
176 
177   FILE* put_body_ = nullptr;
178 
179   StringPiece post_body_buffer_;
180   size_t post_body_read_ = 0;
181 
182   std::vector<char>* response_buffer_ = nullptr;
183 
184   struct DirectResponseState {
185     char* buffer_;
186     size_t buffer_size_;
187     size_t bytes_transferred_;
188     size_t bytes_received_;
189   };
190   DirectResponseState direct_response_ = {};
191 
192   CURL* curl_ = nullptr;
193   curl_slist* curl_headers_ = nullptr;
194   curl_slist* resolve_list_ = nullptr;
195 
196   RequestStats* stats_ = nullptr;
197 
198   std::vector<char> default_response_buffer_;
199 
200   std::unordered_map<string, string> response_headers_;
201   uint64 response_code_ = 0;
202 
203   // The timestamp of the last activity related to the request execution, in
204   // seconds since epoch.
205   uint64 last_progress_timestamp_ = 0;
206   // The last progress in terms of bytes transmitted.
207   curl_off_t last_progress_bytes_ = 0;
208 
209   // The maximum period of request inactivity.
210   uint32 inactivity_timeout_secs_ = 60;  // 1 minute
211 
212   // Timeout for the connection phase.
213   uint32 connect_timeout_secs_ = 120;  // 2 minutes
214 
215   // Tiemout for the whole request. Set only to prevent hanging indefinitely.
216   uint32 request_timeout_secs_ = 3600;  // 1 hour
217 
218   // Members to enforce the usage flow.
219   bool is_uri_set_ = false;
220   bool is_method_set_ = false;
221   bool is_sent_ = false;
222 
223   // Store the URI to help disambiguate requests when errors occur.
224   string uri_;
225   RequestMethod method_ = RequestMethod::kGet;
226 
227   // Limit the size of a http response that is copied into an error message.
228   const size_t response_to_error_limit_ = 500;
229 
230   TF_DISALLOW_COPY_AND_ASSIGN(CurlHttpRequest);
231 };
232 
233 /// \brief A proxy to the libcurl C interface as a dependency injection measure.
234 ///
235 /// This class is meant as a very thin wrapper for the libcurl C library.
236 class LibCurl {
237  public:
~LibCurl()238   virtual ~LibCurl() {}
239 
240   virtual CURL* curl_easy_init() = 0;
241   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
242                                     uint64 param) TF_MUST_USE_RESULT = 0;
243   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
244                                     const char* param) TF_MUST_USE_RESULT = 0;
245   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
246                                     void* param) TF_MUST_USE_RESULT = 0;
247   virtual CURLcode curl_easy_setopt(
248       CURL* curl, CURLoption option,
249       size_t (*param)(void*, size_t, size_t, FILE*)) TF_MUST_USE_RESULT = 0;
250   virtual CURLcode curl_easy_setopt(CURL* curl, CURLoption option,
251                                     size_t (*param)(const void*, size_t, size_t,
252                                                     void*))
253       TF_MUST_USE_RESULT = 0;
254   virtual CURLcode curl_easy_setopt(
255       CURL* curl, CURLoption option,
256       int (*param)(void* clientp, curl_off_t dltotal, curl_off_t dlnow,
257                    curl_off_t ultotal,
258                    curl_off_t ulnow)) TF_MUST_USE_RESULT = 0;
259   virtual CURLcode curl_easy_perform(CURL* curl) TF_MUST_USE_RESULT = 0;
260   virtual CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
261                                      uint64* value) TF_MUST_USE_RESULT = 0;
262   virtual CURLcode curl_easy_getinfo(CURL* curl, CURLINFO info,
263                                      double* value) TF_MUST_USE_RESULT = 0;
264   virtual void curl_easy_cleanup(CURL* curl) = 0;
265   virtual curl_slist* curl_slist_append(curl_slist* list, const char* str) = 0;
266   virtual void curl_slist_free_all(curl_slist* list) = 0;
267   virtual char* curl_easy_escape(CURL* curl, const char* str, int length) = 0;
268   virtual void curl_free(void* p) = 0;
269 };
270 
271 }  // namespace tensorflow
272 
273 #endif  // TENSORFLOW_CORE_PLATFORM_CLOUD_CURL_HTTP_REQUEST_H_
274