Skip to content

Commit af86c93

Browse files
andreasdrandreasdr
committed
network: HTTPClient+HTTPDownloadClientTest: some improvements
1 parent f6dfc16 commit af86c93

File tree

7 files changed

+197
-64
lines changed

7 files changed

+197
-64
lines changed

‎src/tdme/network/httpclient/HTTPClient.cpp

+24-11
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ const string HTTPClient::HTTP_METHOD_PUT = "PUT";
5151
const string HTTPClient::HTTP_METHOD_DELETE = "DELETE";
5252

5353
string HTTPClient::urlEncode(const string &value) {
54+
// TODO: put me into utilities
5455
// see: https://stackoverflow.com/questions/154536/encode-decode-urls-in-c
5556
ostringstream escaped;
5657
escaped.fill('0');
@@ -74,8 +75,7 @@ string HTTPClient::urlEncode(const string &value) {
7475
return escaped.str();
7576
}
7677

77-
78-
string HTTPClient::createHTTPRequestHeaders(const string& hostname, const string& method, const string& relativeUrl, const unordered_map<string, string>& getParameters, const unordered_map<string, string>& postParameters, const string& body) {
78+
string HTTPClient::createHTTPRequestHeaders(const string& hostname, const string& relativeUrl, const string& body) {
7979
string query;
8080
for (const auto& [parameterName, parameterValue]: getParameters) {
8181
if (query.empty() == true) query+= "?"; else query+="&";
@@ -96,6 +96,9 @@ string HTTPClient::createHTTPRequestHeaders(const string& hostname, const string
9696
request+=
9797
string("Content-Type: " + contentType + "\r\n");
9898
}
99+
for (const auto& [headerName, headerValue]: headers) {
100+
request+= headerName + ": " + headerValue + "\r\n";
101+
}
99102
if (method == HTTP_METHOD_POST || method == HTTP_METHOD_PUT) {
100103
string _body;
101104
if (postParameters.size() > 0) {
@@ -114,15 +117,24 @@ string HTTPClient::createHTTPRequestHeaders(const string& hostname, const string
114117
return request;
115118
}
116119

117-
void HTTPClient::parseHTTPResponseHeaders(stringstream& rawResponse, int16_t& httpStatusCode, vector<string>& httpHeader) {
120+
void HTTPClient::parseHTTPResponseHeaders(stringstream& rawResponse, int16_t& statusCode, unordered_map<string, string>& responseHeaders) {
121+
int headerIdx = 0;
122+
string statusHeader;
118123
string line;
119124
char lastChar = -1;
120125
char currentChar;
121126
while (rawResponse.eof() == false) {
122127
rawResponse.get(currentChar);
123128
if (lastChar == '\r' && currentChar == '\n') {
124129
if (line.empty() == false) {
125-
httpHeader.push_back(line);
130+
if (headerIdx == 0) {
131+
statusHeader = line;
132+
headerIdx++;
133+
} else {
134+
auto headerNameValueSeparator = StringTools::indexOf(line, ':');
135+
responseHeaders[StringTools::trim(StringTools::substring(line, 0, headerNameValueSeparator))] =
136+
StringTools::trim(StringTools::substring(line, headerNameValueSeparator + 1));
137+
}
126138
} else {
127139
break;
128140
}
@@ -133,13 +145,13 @@ void HTTPClient::parseHTTPResponseHeaders(stringstream& rawResponse, int16_t& ht
133145
}
134146
lastChar = currentChar;
135147
}
136-
if (httpHeader.size() > 0) {
148+
if (statusHeader.empty() == false) {
137149
StringTokenizer t;
138-
t.tokenize(httpHeader[0], " ");
150+
t.tokenize(statusHeader, " ");
139151
for (auto i = 0; i < 3 && t.hasMoreTokens(); i++) {
140152
auto token = t.nextToken();
141153
if (i == 1) {
142-
httpStatusCode = Integer::parse(token);
154+
statusCode = Integer::parse(token);
143155
}
144156
}
145157
}
@@ -148,14 +160,15 @@ void HTTPClient::parseHTTPResponseHeaders(stringstream& rawResponse, int16_t& ht
148160
void HTTPClient::reset() {
149161
url.clear();
150162
method.clear();
163+
headers.clear();
151164
getParameters.clear();
152165
postParameters.clear();
153166
body.clear();
154167
contentType.clear();
155168

156169
rawResponse.clear();
157-
httpStatusCode = -1;
158-
httpHeader.clear();
170+
statusCode = -1;
171+
responseHeaders.clear();
159172
}
160173

161174
void HTTPClient::execute() {
@@ -183,7 +196,7 @@ void HTTPClient::execute() {
183196

184197
TCPSocket::create(socket, TCPSocket::determineIpVersion(ip));
185198
socket.connect(ip, 80);
186-
auto request = createHTTPRequestHeaders(hostname, method, relativeUrl, getParameters, postParameters, body);
199+
auto request = createHTTPRequestHeaders(hostname, relativeUrl, body);
187200
socket.write((void*)request.data(), request.length());
188201

189202
char rawResponseBuf[16384];
@@ -198,7 +211,7 @@ void HTTPClient::execute() {
198211
}
199212

200213
//
201-
parseHTTPResponseHeaders(rawResponse, httpStatusCode, httpHeader);
214+
parseHTTPResponseHeaders(rawResponse, statusCode, responseHeaders);
202215

203216
//
204217
socket.shutdown();

‎src/tdme/network/httpclient/HTTPClient.h

+34-19
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ class tdme::network::httpclient::HTTPClient {
2626
private:
2727
string url;
2828
string method;
29+
unordered_map<string, string> headers;
2930
unordered_map<string, string> getParameters;
3031
unordered_map<string, string> postParameters;
3132
string body;
@@ -34,27 +35,31 @@ class tdme::network::httpclient::HTTPClient {
3435
string password;
3536

3637
stringstream rawResponse;
37-
int16_t httpStatusCode { -1 };
38-
vector<string> httpHeader;
38+
int16_t statusCode { -1 };
39+
unordered_map<string, string> responseHeaders;
40+
41+
/**
42+
* Returns a URL encoded representation of value
43+
* @param value value
44+
* @return URL encoded value
45+
*/
46+
static string urlEncode(const string& value);
3947

4048
/**
4149
* Create HTTP request headers
4250
* @param hostname hostname
43-
* @param method method
4451
* @param relativeUrl url relative to server root
45-
* @param getParameters get parameters
46-
* @param postParameter post parameters
4752
* @param body body
4853
*/
49-
string createHTTPRequestHeaders(const string& hostname, const string& method, const string& relativeUrl, const unordered_map<string, string>& getParameters, const unordered_map<string, string>& postParameters, const string& body);
54+
string createHTTPRequestHeaders(const string& hostname, const string& relativeUrl, const string& body);
5055

5156
/**
5257
* Parse HTTP response headers
5358
* @param rawResponse raw response
54-
* @param httpStatusCode HTTP status code
55-
* @param httpHeader HTTP header
59+
* @param statusCode HTTP status code
60+
* @param responseHeaders HTTP response headers
5661
*/
57-
void parseHTTPResponseHeaders(stringstream& rawResponse, int16_t& httpStatusCode, vector<string>& httpHeader);
62+
void parseHTTPResponseHeaders(stringstream& rawResponse, int16_t& statusCode, unordered_map<string, string>& responseHeaders);
5863

5964
public:
6065
static const constexpr int16_t HTTP_STATUSCODE_OK { 200 };
@@ -129,6 +134,22 @@ class tdme::network::httpclient::HTTPClient {
129134
this->password = password;
130135
}
131136

137+
/**
138+
* Get headers
139+
* @return headers
140+
*/
141+
inline const unordered_map<string, string>& getHeaders() {
142+
return headers;
143+
}
144+
145+
/**
146+
* Set headers
147+
* @param headers headers
148+
*/
149+
inline void setHeaders(const unordered_map<string, string>& headers) {
150+
this->headers = headers;
151+
}
152+
132153
/**
133154
* Get GET parameter
134155
* @return GET parameter
@@ -219,20 +240,14 @@ class tdme::network::httpclient::HTTPClient {
219240
* @return HTTP status code
220241
*/
221242
inline int16_t getStatusCode() {
222-
return httpStatusCode;
243+
return statusCode;
223244
}
224245

225246
/**
226-
* @return HTTP response headers
247+
* @return response headers
227248
*/
228-
inline const vector<string>& getResponseHeaders() {
229-
return httpHeader;
249+
inline const unordered_map<string, string>& getResponseHeaders() {
250+
return responseHeaders;
230251
}
231252

232-
/**
233-
* Returns a URL encoded representation of value
234-
* @return URL encoded value
235-
*/
236-
static string urlEncode(const string& value);
237-
238253
};

‎src/tdme/network/httpclient/HTTPDownloadClient.cpp

+74-19
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
#include <tdme/network/httpclient/HTTPDownloadClient.h>
22

3+
#include <iomanip>
34
#include <memory>
45
#include <filesystem>
56
#include <fstream>
67
#include <string>
8+
#include <sstream>
9+
#include <unordered_map>
710
#include <vector>
811

912
#include <tdme/tdme.h>
@@ -24,13 +27,19 @@
2427
#include <tdme/utilities/StringTokenizer.h>
2528
#include <tdme/utilities/StringTools.h>
2629

30+
using std::hex;
2731
using std::make_unique;
32+
using std::nouppercase;
2833
using std::ifstream;
2934
using std::ios;
3035
using std::ofstream;
36+
using std::ostringstream;
37+
using std::setw;
3138
using std::string;
3239
using std::to_string;
3340
using std::unique_ptr;
41+
using std::unordered_map;
42+
using std::uppercase;
3443
using std::vector;
3544

3645
using tdme::math::Math;
@@ -55,9 +64,39 @@ using tdme::network::httpclient::HTTPDownloadClient;
5564
HTTPDownloadClient::HTTPDownloadClient(): downloadThreadMutex("downloadthread-mutex") {
5665
}
5766

67+
string HTTPDownloadClient::urlEncode(const string &value) {
68+
// TODO: put me into utilities
69+
// see: https://stackoverflow.com/questions/154536/encode-decode-urls-in-c
70+
ostringstream escaped;
71+
escaped.fill('0');
72+
escaped << hex;
73+
74+
for (string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
75+
string::value_type c = (*i);
76+
77+
// Keep alphanumeric and other accepted characters intact
78+
if (Character::isAlphaNumeric(c) == true || c == '-' || c == '_' || c == '.' || c == '~') {
79+
escaped << c;
80+
continue;
81+
}
82+
83+
// Any other characters are percent-encoded
84+
escaped << uppercase;
85+
escaped << '%' << setw(2) << int((unsigned char) c);
86+
escaped << nouppercase;
87+
}
88+
89+
return escaped.str();
90+
}
91+
5892
string HTTPDownloadClient::createHTTPRequestHeaders(const string& hostName, const string& relativeUrl) {
93+
string query;
94+
for (const auto& [parameterName, parameterValue]: getParameters) {
95+
if (query.empty() == true) query+= "?"; else query+="&";
96+
query+= urlEncode(parameterName) + "=" + urlEncode(parameterValue);
97+
}
5998
auto request =
60-
string("GET " + relativeUrl + " HTTP/1.1\r\n") +
99+
string("GET " + relativeUrl + query + " HTTP/1.1\r\n") +
61100
string("User-Agent: tdme2-httpdownloadclient\r\n") +
62101
string("Host: " + hostName + "\r\n") +
63102
string("Connection: close\r\n");
@@ -66,24 +105,36 @@ string HTTPDownloadClient::createHTTPRequestHeaders(const string& hostName, cons
66105
Base64::encode(username + ":" + password, base64Pass);
67106
request+= "Authorization: Basic " + base64Pass + "\r\n";
68107
}
108+
for (const auto& [headerName, headerValue]: headers) {
109+
request+= headerName + ": " + headerValue + "\r\n";
110+
}
69111
request+=
70112
string("\r\n");
71113
return request;
72114
}
73115

74-
uint64_t HTTPDownloadClient::parseHTTPResponseHeaders(ifstream& rawResponse, int16_t& httpStatusCode, vector<string>& httpHeader) {
75-
httpHeader.clear();
116+
uint64_t HTTPDownloadClient::parseHTTPResponseHeaders(ifstream& rawResponse, int16_t& statusCode, unordered_map<string, string>& responseHeaders) {
117+
responseHeaders.clear();
118+
auto headerSize = 0ll;
119+
auto returnHeaderSize = 0ll;
120+
int headerIdx = 0;
121+
string statusHeader;
76122
string line;
77-
uint64_t headerSize = 0;
78-
uint64_t returnHeaderSize = 0;
79123
char lastChar = -1;
80124
char currentChar;
81125
while (rawResponse.eof() == false) {
82126
rawResponse.get(currentChar);
83127
headerSize++;
84128
if (lastChar == '\r' && currentChar == '\n') {
85129
if (line.empty() == false) {
86-
httpHeader.push_back(line);
130+
if (headerIdx == 0) {
131+
statusHeader = line;
132+
headerIdx++;
133+
} else {
134+
auto headerNameValueSeparator = StringTools::indexOf(line, ':');
135+
responseHeaders[StringTools::trim(StringTools::substring(line, 0, headerNameValueSeparator))] =
136+
StringTools::trim(StringTools::substring(line, headerNameValueSeparator + 1));
137+
}
87138
} else {
88139
returnHeaderSize = headerSize;
89140
break;
@@ -95,24 +146,28 @@ uint64_t HTTPDownloadClient::parseHTTPResponseHeaders(ifstream& rawResponse, int
95146
}
96147
lastChar = currentChar;
97148
}
98-
if (httpHeader.size() > 0) {
149+
if (statusHeader.empty() == false) {
99150
StringTokenizer t;
100-
t.tokenize(httpHeader[0], " ");
151+
t.tokenize(statusHeader, " ");
101152
for (auto i = 0; i < 3 && t.hasMoreTokens(); i++) {
102153
auto token = t.nextToken();
103154
if (i == 1) {
104-
httpStatusCode = Integer::parse(token);
155+
statusCode = Integer::parse(token);
105156
}
106157
}
107158
}
159+
//
108160
return returnHeaderSize;
109161
}
110162

111163
void HTTPDownloadClient::reset() {
112164
url.clear();
113165
file.clear();
114-
httpStatusCode = -1;
115-
httpHeader.clear();
166+
headers.clear();
167+
getParameters.clear();
168+
statusCode = -1;
169+
responseHeaders.clear();
170+
//
116171
haveHeaders = false;
117172
haveContentSize = false;
118173
headerSize = 0LL;
@@ -180,14 +235,14 @@ void HTTPDownloadClient::start() {
180235
throw HTTPClientException("Unable to open file for reading(" + to_string(errno) + "): " + (downloadClient->file + ".download"));
181236
}
182237
// try to read headers
183-
downloadClient->httpHeader.clear();
184-
if ((downloadClient->headerSize = downloadClient->parseHTTPResponseHeaders(ifs, downloadClient->httpStatusCode, downloadClient->httpHeader)) > 0) {
238+
downloadClient->responseHeaders.clear();
239+
if ((downloadClient->headerSize = downloadClient->parseHTTPResponseHeaders(ifs, downloadClient->statusCode, downloadClient->responseHeaders)) > 0) {
185240
downloadClient->haveHeaders = true;
186-
for (const auto& header: downloadClient->httpHeader) {
187-
if (StringTools::startsWith(header, "Content-Length: ") == true) {
188-
downloadClient->haveContentSize = true;
189-
downloadClient->contentSize = Integer::parse(StringTools::substring(header, string("Content-Length: ").size()));
190-
}
241+
auto contentLengthHeaderIt = downloadClient->responseHeaders.find("Content-Length");
242+
if (contentLengthHeaderIt != downloadClient->responseHeaders.end()) {
243+
const auto& contentLengthHeader = contentLengthHeaderIt->second;
244+
downloadClient->haveContentSize = true;
245+
downloadClient->contentSize = Integer::parse(contentLengthHeader);
191246
}
192247
}
193248
ifs.close();
@@ -206,7 +261,7 @@ void HTTPDownloadClient::start() {
206261
}
207262

208263
// transfer to real file
209-
if (downloadClient->httpStatusCode == 200 && isStopRequested() == false) {
264+
if (downloadClient->statusCode == 200 && isStopRequested() == false) {
210265
// input file stream
211266
ifstream ifs(std::filesystem::u8path(downloadClient->file + ".download"), ofstream::binary);
212267
if (ifs.is_open() == false) {

0 commit comments

Comments
 (0)