support archive.org protected urls

CMakeLists.txt

@@ -69,7 +69,7 @@ add_executable(ezremote_client
 
 add_self(ezremote_client)
 
-add_pkg(ezremote_client ${CMAKE_SOURCE_DIR}/data "RMTC00001" "ezRemote Client" "01.32" 32 0)
+add_pkg(ezremote_client ${CMAKE_SOURCE_DIR}/data "RMTC00001" "ezRemote Client" "01.34" 32 0)
 
 target_link_libraries(ezremote_client
   c

source/clients/archiveorg.cpp

@@ -15,7 +15,119 @@ using httplib::Client;
 using httplib::Headers;
 using httplib::Result;
 
+struct InsensitiveCompare
+{
+    bool operator()(const std::string &a, const std::string &b) const
+    {
+        return strcasecmp(a.c_str(), b.c_str()) < 0;
+    }
+};
+
 static std::map<std::string, int> month_map = {{"Jan", 1}, {"Feb", 2}, {"Mar", 3}, {"Apr", 4}, {"May", 5}, {"Jun", 6}, {"Jul", 7}, {"Aug", 8}, {"Sep", 9}, {"Oct", 10}, {"Nov", 11}, {"Dec", 12}};
+static std::set<std::string, InsensitiveCompare> ignore_cookie_keys = {"path", "expires", "max-age", "domain", "secure"};
+
+std::string ArchiveOrgClient::GenerateRandomId(const int len)
+{
+    static const char alphanum[] = "0123456789abcdef";
+    std::string tmp_s;
+    tmp_s.reserve(len);
+
+    for (int i = 0; i < len; ++i) {
+        tmp_s += alphanum[rand() % (sizeof(alphanum) - 1)];
+    }
+    
+    return tmp_s;
+}
+
+int ArchiveOrgClient::Connect(const std::string &url, const std::string &username, const std::string &password)
+{
+    this->host_url = url;
+    size_t scheme_pos = url.find("://");
+    size_t root_pos = url.find("/", scheme_pos + 3);
+    if (root_pos != std::string::npos)
+    {
+        this->host_url = url.substr(0, root_pos);
+        this->base_path = url.substr(root_pos);
+    }
+    client = new httplib::Client(this->host_url);
+    client->set_keep_alive(true);
+    client->set_follow_location(true);
+    client->set_connection_timeout(30);
+    client->set_read_timeout(30);
+    client->enable_server_certificate_verification(false);
+
+    this->cookies = {
+        {"donation-identifier", GenerateRandomId(32)},
+        {"test-cookie", "1"},
+        {"abtest-identifier", GenerateRandomId(32)}
+    };
+
+    if (username.length() > 0)
+        return Login(username, password);
+    else if (Ping())
+        this->connected = true;
+    return 1;
+}
+
+int ArchiveOrgClient::Login(const std::string &username, const std::string &password)
+{
+    std::string url = std::string("/account/login");
+    Headers headers = {{ "User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"}};
+    SetCookies(headers);
+
+    MultipartFormDataItems items = {
+        {"username", username, "", ""},
+        {"password", password, "", ""},
+        {"remember", "true", "", ""},
+        {"referer", "https://archive.org/", "", ""},
+        {"login", "true", "", ""},
+        {"submit_by_js", "true", "", ""}};
+
+    if (auto res = client->Post(url, headers, items))
+    {
+        if (HTTP_SUCCESS(res->status))
+        {
+            if (res->has_header("Set-Cookie"))
+            {
+                int cookies_count = res->get_header_value_count("Set-Cookie");
+
+                for (int i = 0; i < cookies_count; i++)
+                {
+                    std::string cookie_str = res->get_header_value("Set-Cookie", i);
+
+                    std::vector<std::string> cookies = Util::Split(cookie_str, ";");
+                    for (std::vector<std::string>::iterator it = cookies.begin(); it != cookies.end();)
+                    {
+                        std::vector<std::string> cookie = Util::Split(*it, "=");
+                        std::string key = Util::Trim(cookie[0], " ");
+                        if (ignore_cookie_keys.find(key) == ignore_cookie_keys.end())
+                        {
+                            if (cookie.size() > 1)
+                                this->cookies[key] = Util::Trim(cookie[1], " ");
+                            else
+                                this->cookies[key] = "";
+                        }
+                        ++it;
+                    }
+                }
+                this->connected = true;
+                return 1;
+            }
+            else
+            {
+                return 0;
+            }
+        }
+        else
+        {
+            return 0;
+        }
+    }
+    else
+    {
+        return 0;
+    }
+}
 
 std::vector<DirEntry> ArchiveOrgClient::ListDir(const std::string &path)
 {
@@ -23,9 +135,11 @@ std::vector<DirEntry> ArchiveOrgClient::ListDir(const std::string &path)
     DirEntry entry;
     Util::SetupPreviousFolder(path, &entry);
     out.push_back(entry);
+    Headers headers;
+    SetCookies(headers);
 
     std::string encoded_path = httplib::detail::encode_url(GetFullPath(path) + "/");
-    if (auto res = client->Get(encoded_path))
+    if (auto res = client->Get(encoded_path, headers))
     {
         lxb_status_t status;
         lxb_dom_attr_t *attr;
@@ -130,6 +244,13 @@ std::vector<DirEntry> ArchiveOrgClient::ListDir(const std::string &path)
             // td0 contains the <a> tag
             td_element = lxb_dom_collection_element(td_collection, 0);
             lxb_dom_node_t *a_node = NextChildElement(td_element);
+            // there is no a_node in protected links
+            if (a_node == nullptr)
+            {
+                lxb_dom_collection_destroy(td_collection, true);
+                continue;
+            }
+
             value = lxb_dom_element_local_name(lxb_dom_interface_element(a_node), &value_len);
             tmp_string = std::string((const char *)value, value_len);
             if (tmp_string.compare("a") != 0)
@@ -236,4 +357,4 @@ std::vector<DirEntry> ArchiveOrgClient::ListDir(const std::string &path)
 
 finish:
     return out;
-}
+}

source/clients/archiveorg.h

@@ -11,7 +11,12 @@
 class ArchiveOrgClient : public BaseClient
 {
 public:
+    int Connect(const std::string &url, const std::string &username, const std::string &password);
     std::vector<DirEntry> ListDir(const std::string &path);
+
+private:
+    int Login(const std::string &username, const std::string &password);
+    std::string GenerateRandomId(const int len);
 };
 
-#endif
+#endif

source/clients/baseclient.cpp

@@ -20,6 +20,26 @@ BaseClient::~BaseClient()
         delete client;
 };
 
+int BaseClient::SetCookies(Headers &headers)
+{
+    if (this->cookies.size() > 0)
+    {
+        std::string cookie;
+        for (std::map<std::string, std::string>::iterator it = this->cookies.begin(); it != this->cookies.end();)
+        {
+            cookie.append(it->first).append("=").append(it->second);
+            if (std::next(it, 1) != this->cookies.end())
+            {
+                cookie.append("; ");
+            }
+            ++it;
+        }
+        headers.emplace("Cookie", cookie);
+    }
+
+    return 1;
+}
+
 int BaseClient::Connect(const std::string &url, const std::string &username, const std::string &password)
 {
     this->host_url = url;
@@ -81,7 +101,10 @@ int BaseClient::Rmdir(const std::string &path, bool recursive)
 
 int BaseClient::Size(const std::string &path, int64_t *size)
 {
-    if (auto res = client->Head(GetFullPath(path)))
+    Headers headers;
+    SetCookies(headers);
+
+    if (auto res = client->Head(GetFullPath(path), headers))
     {
         if (HTTP_SUCCESS(res->status))
         {
@@ -94,6 +117,8 @@ int BaseClient::Size(const std::string &path, int64_t *size)
              // example: Content-Range: bytes 0-10/4372785
         {
             Headers headers = {{"Range", "bytes=0-1"}};
+            SetCookies(headers);
+
             if (auto range_res = client->Get(GetFullPath(path), headers))
             {
                 if (HTTP_SUCCESS(range_res->status))
@@ -113,7 +138,7 @@ int BaseClient::Size(const std::string &path, int64_t *size)
         }
         else
         {
-            sprintf(this->response, "%d - %s", res->status, detail::status_message(res->status));
+            sprintf(this->response, "%d - %s", res->status, http_status_message(res->status));
         }
     }
     else
@@ -128,8 +153,10 @@ int BaseClient::Get(const std::string &outputfile, const std::string &path, uint
     std::ofstream file_stream(outputfile, std::ios::binary);
     bytes_transfered = 0;
     sceRtcGetCurrentTick(&prev_tick);
+    Headers headers;
+    SetCookies(headers);
 
-    if (auto res = client->Get(GetFullPath(path),
+    if (auto res = client->Get(GetFullPath(path), headers,
                                [&](const char *data, size_t data_length)
                                {
                                    file_stream.write(data, data_length);
@@ -149,7 +176,10 @@ int BaseClient::Get(const std::string &outputfile, const std::string &path, uint
 
 int BaseClient::Get(SplitFile *split_file, const std::string &path, uint64_t offset)
 {
-    if (auto res = client->Get(GetFullPath(path),
+    Headers headers;
+    SetCookies(headers);
+
+    if (auto res = client->Get(GetFullPath(path), headers,
                                [&](const char *data, size_t data_length)
                                {
                                    if (!split_file->IsClosed())
@@ -175,6 +205,8 @@ int BaseClient::GetRange(const std::string &path, DataSink &sink, uint64_t size,
     char range_header[64];
     sprintf(range_header, "bytes=%lu-%lu", offset, offset + size - 1);
     Headers headers = {{"Range", range_header}};
+    SetCookies(headers);
+
     size_t bytes_read = 0;
     if (auto res = client->Get(GetFullPath(path), headers,
                                [&](const char *data, size_t data_length)
@@ -198,6 +230,8 @@ int BaseClient::GetRange(const std::string &path, void *buffer, uint64_t size, u
     char range_header[64];
     sprintf(range_header, "bytes=%lu-%lu", offset, offset + size - 1);
     Headers headers = {{"Range", range_header}};
+    SetCookies(headers);
+
     size_t bytes_read = 0;
     std::vector<char> body;
     if (auto res = client->Get(GetFullPath(path), headers,
@@ -257,6 +291,8 @@ int BaseClient::Head(const std::string &path, void *buffer, uint64_t len)
     char range_header[64];
     sprintf(range_header, "bytes=%lu-%lu", 0L, len - 1);
     Headers headers = {{"Range", range_header}};
+    SetCookies(headers);
+
     size_t bytes_read = 0;
     std::vector<char> body;
     if (auto res = client->Get(GetFullPath(path), headers,

source/clients/baseclient.h

@@ -49,11 +49,14 @@ public:
     static std::string UnEscape(const std::string &url);
 
 protected:
+    int SetCookies(httplib::Headers &headers);
+
     httplib::Client *client;
     std::string base_path;
     std::string host_url;
     char response[512];
     bool connected = false;
+    std::map<std::string, std::string> cookies;
 };
 
 #endif

source/clients/ftpclient.cpp

@@ -24,8 +24,6 @@
 #define FTP_CLIENT_READ 1
 #define FTP_CLIENT_WRITE 2
 
-#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y))
-
 FtpClient::FtpClient()
 {
 	mp_ftphandle = static_cast<ftphandle *>(calloc(1, sizeof(ftphandle)));
@@ -125,7 +123,7 @@ int FtpClient::Connect(const std::string &url, const std::string &user, const st
 	}
 	mp_ftphandle->handle = sControl;
 
-	if (ReadResponse('2', mp_ftphandle) == 0)
+	if (ReadResponse("2", mp_ftphandle) == 0)
 	{
 		close(mp_ftphandle->handle);
 		mp_ftphandle->handle = 0;
@@ -142,7 +140,7 @@ int FtpClient::Connect(const std::string &url, const std::string &user, const st
 		cmd = "USER anonymous";
 	}
 
-	if (!FtpSendCmd(cmd, '3', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "3", mp_ftphandle))
 	{
 		if (mp_ftphandle->ctrl != NULL)
 			return 1;
@@ -161,7 +159,7 @@ int FtpClient::Connect(const std::string &url, const std::string &user, const st
 
 	cmd = "PASS " + pass;
 	int ret;
-	if ((ret = FtpSendCmd(cmd, '2', mp_ftphandle)))
+	if ((ret = FtpSendCmd(cmd, "2", mp_ftphandle)))
 	{
 		mp_ftphandle->is_connected = true;
 	}
@@ -179,7 +177,7 @@ int FtpClient::Connect(const std::string &url, const std::string &user, const st
  *
  * return 1 if proper response received, 0 otherwise
  */
-int FtpClient::FtpSendCmd(const std::string &cmd, char expected_resp, ftphandle *nControl)
+int FtpClient::FtpSendCmd(const std::string &cmd, const std::string &expected_resp, ftphandle *nControl)
 {
 	char buf[512];
 	int x;
@@ -205,7 +203,7 @@ int FtpClient::FtpSendCmd(const std::string &cmd, char expected_resp, ftphandle
  * return 0 if first char doesn't match
  * return 1 if first char matches
  */
-int FtpClient::ReadResponse(char c, ftphandle *nControl)
+int FtpClient::ReadResponse(const std::string &c, ftphandle *nControl)
 {
 	char match[5];
 
@@ -228,7 +226,7 @@ int FtpClient::ReadResponse(char c, ftphandle *nControl)
 		} while (strncmp(nControl->response, match, 4));
 	}
 
-	if (nControl->response[0] == c)
+	if (c.find(nControl->response[0]) != std::string::npos)
 		return 1;
 	return 0;
 }
@@ -368,7 +366,7 @@ int FtpClient::FtpAccess(const std::string &path, accesstype type, transfermode
 		return 0;
 	}
 	sprintf(buf, "TYPE %c", mode);
-	if (!FtpSendCmd(buf, '2', nControl))
+	if (!FtpSendCmd(buf, "2", nControl))
 		return 0;
 
 	switch (type)
@@ -477,7 +475,7 @@ int FtpClient::FtpAcceptConnection(ftphandle *nData, ftphandle *nControl)
 	{
 		close(nData->handle);
 		nData->handle = 0;
-		ReadResponse('2', nControl);
+		ReadResponse("2", nControl);
 		rv = 0;
 	}
 
@@ -521,7 +519,7 @@ int FtpClient::FtpOpenPasv(ftphandle *nControl, ftphandle **nData, transfermode
 
 	memset(&sin, 0, l);
 	sin.in.sin_family = AF_INET;
-	if (!FtpSendCmd("PASV", '2', nControl))
+	if (!FtpSendCmd("PASV", "2", nControl))
 		return -1;
 	cp = strchr(nControl->response, '(');
 	if (cp == NULL)
@@ -542,7 +540,7 @@ int FtpClient::FtpOpenPasv(ftphandle *nControl, ftphandle **nData, transfermode
 	{
 		char buf[512];
 		sprintf(buf, "REST %lld", mp_ftphandle->offset);
-		if (!FtpSendCmd(buf, '3', nControl))
+		if (!FtpSendCmd(buf, "3", nControl))
 			return 0;
 	}
 
@@ -592,7 +590,7 @@ int FtpClient::FtpOpenPasv(ftphandle *nControl, ftphandle **nData, transfermode
 		return -1;
 	}
 
-	if (!ReadResponse('1', nControl))
+	if (!ReadResponse("1", nControl))
 	{
 		close(sData);
 		return -1;
@@ -717,7 +715,7 @@ int FtpClient::FtpOpenPort(ftphandle *nControl, ftphandle **nData, transfermode
 			(unsigned char)sin.sa.sa_data[5],
 			(unsigned char)sin.sa.sa_data[0],
 			(unsigned char)sin.sa.sa_data[1]);
-	if (!FtpSendCmd(buf, '2', nControl))
+	if (!FtpSendCmd(buf, "2", nControl))
 	{
 		close(sData);
 		return -1;
@@ -727,7 +725,7 @@ int FtpClient::FtpOpenPort(ftphandle *nControl, ftphandle **nData, transfermode
 	{
 		char buf[512];
 		sprintf(buf, "REST %lld", mp_ftphandle->offset);
-		if (!FtpSendCmd(buf, '3', nControl))
+		if (!FtpSendCmd(buf, "3", nControl))
 		{
 			close(sData);
 			return 0;
@@ -747,7 +745,7 @@ int FtpClient::FtpOpenPort(ftphandle *nControl, ftphandle **nData, transfermode
 		return -1;
 	}
 
-	if (!FtpSendCmd(cmd, '1', nControl))
+	if (!FtpSendCmd(cmd, "1", nControl))
 	{
 		FtpClose(*nData);
 		*nData = NULL;
@@ -1045,7 +1043,7 @@ int FtpClient::FtpClose(ftphandle *nData)
 	ctrl = nData->ctrl;
 	free(nData);
 	if (ctrl)
-		return ReadResponse('2', ctrl);
+		return ReadResponse("2", ctrl);
 	return 1;
 }
 
@@ -1061,7 +1059,7 @@ int FtpClient::Quit()
 		strcpy(mp_ftphandle->response, "error: no anwser from server\n");
 		return 0;
 	}
-	FtpSendCmd("QUIT", '2', mp_ftphandle);
+	FtpSendCmd("QUIT", "2", mp_ftphandle);
 	shutdown(mp_ftphandle->handle, SHUT_WR);
 	struct linger lng = {1, 0};
 	setsockopt(mp_ftphandle->handle, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng));
@@ -1105,7 +1103,7 @@ int FtpClient::RawRead(void *buf, int max, ftphandle *handle)
 int FtpClient::Site(const std::string &cmd)
 {
 	std::string tmp = "SITE " + cmd;
-	if (!FtpSendCmd(tmp, '2', mp_ftphandle))
+	if (!FtpSendCmd(tmp, "2", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1118,7 +1116,7 @@ int FtpClient::Site(const std::string &cmd)
 
 int FtpClient::Raw(const std::string &cmd)
 {
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1137,7 +1135,7 @@ int FtpClient::SysType(char *buf, int max)
 	int l = max;
 	char *b = buf;
 	char *s;
-	if (!FtpSendCmd("SYST", '2', mp_ftphandle))
+	if (!FtpSendCmd("SYST", "2", mp_ftphandle))
 		return 0;
 	s = &mp_ftphandle->response[4];
 	while ((--l) && (*s != ' '))
@@ -1154,7 +1152,7 @@ int FtpClient::SysType(char *buf, int max)
 int FtpClient::Mkdir(const std::string &path)
 {
 	std::string cmd = "MKD " + path;
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1167,7 +1165,7 @@ int FtpClient::Mkdir(const std::string &path)
 int FtpClient::Chdir(const std::string &path)
 {
 	std::string cmd = "CWD " + path;
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1179,7 +1177,7 @@ int FtpClient::Chdir(const std::string &path)
  */
 int FtpClient::Cdup()
 {
-	if (!FtpSendCmd("CDUP", '2', mp_ftphandle))
+	if (!FtpSendCmd("CDUP", "2", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1191,7 +1189,7 @@ int FtpClient::Cdup()
  */
 bool FtpClient::Noop()
 {
-	if (!FtpSendCmd("NOOP", '2', mp_ftphandle))
+	if (!FtpSendCmd("NOOP", "25", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1209,7 +1207,7 @@ bool FtpClient::Ping()
 int FtpClient::Rmdir(const std::string &path)
 {
 	std::string cmd = "RMD " + path;
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 	return 1;
 }
@@ -1272,11 +1270,11 @@ int FtpClient::Size(const std::string &path, int64_t *size)
 		return 0;
 
 	sprintf(cmd, "TYPE %c", FtpClient::transfermode::image);
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 
 	sprintf(cmd, "SIZE %s", path.c_str());
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		rv = 0;
 	else
 	{
@@ -1408,10 +1406,10 @@ int FtpClient::Put(const std::string &inputfile, const std::string &path, uint64
 int FtpClient::Rename(const std::string &src, const std::string &dst)
 {
 	std::string cmd = "RNFR " + src;
-	if (!FtpSendCmd(cmd, '3', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "3", mp_ftphandle))
 		return 0;
 	cmd = "RNTO " + dst;
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 
 	return 1;
@@ -1420,7 +1418,7 @@ int FtpClient::Rename(const std::string &src, const std::string &dst)
 int FtpClient::Delete(const std::string &path)
 {
 	std::string cmd = "DELE " + path;
-	if (!FtpSendCmd(cmd, '2', mp_ftphandle))
+	if (!FtpSendCmd(cmd, "2", mp_ftphandle))
 		return 0;
 	return 1;
 }

source/clients/ftpclient.h

@@ -114,12 +114,12 @@ private:
 	char server[128];
 	int server_port;
 
-	int FtpSendCmd(const std::string &cmd, char expected_resp, ftphandle *nControl);
+	int FtpSendCmd(const std::string &cmd, const std::string &expected_resp, ftphandle *nControl);
 	ftphandle *RawOpen(const std::string &path, accesstype type, transfermode mode);
 	int RawClose(ftphandle *handle);
 	int RawWrite(void *buf, int len, ftphandle *handle);
 	int RawRead(void *buf, int max, ftphandle *handle);
-	int ReadResponse(char c, ftphandle *nControl);
+	int ReadResponse(const std::string &c, ftphandle *nControl);
 	int Readline(char *buf, int max, ftphandle *nControl);
 	int Writeline(char *buf, int len, ftphandle *nData);
 	void ClearHandle();