support archive.org protected urls

source/clients/archiveorg.cpp

@@ -17,15 +17,72 @@ using httplib::Result;
 
 static std::map<std::string, int> month_map = {{"Jan", 1}, {"Feb", 2}, {"Mar", 3}, {"Apr", 4}, {"May", 5}, {"Jun", 6}, {"Jul", 7}, {"Aug", 8}, {"Sep", 9}, {"Oct", 10}, {"Nov", 11}, {"Dec", 12}};
 
+int ArchiveOrgClient::Connect(const std::string &url, const std::string &username, const std::string &password)
+{
+    int ret = BaseClient::Connect(url, username, password);
+    if (ret)
+    {
+        return Login(username, password);
+    }
+    return 1;
+}
+
+int ArchiveOrgClient::Login(const std::string &username, const std::string &password)
+{
+    std::string url = std::string("/account/login");
+    std::string post_data = std::string("username=") + username +
+                            "&password=" + password +
+                            "&remember=true" +
+                            "&referer=https://archive.org/" +
+                            "&login=true" +
+                            "&submit_by_js=true";
+
+    if (auto res = client->Post(url, post_data.c_str(), post_data.length(), "application/x-www-form-urlencoded"))
+    {
+        if (HTTP_SUCCESS(res->status))
+        {
+            if (res->has_header("set-cookie"))
+            {
+                int cookies_count = res->get_header_value_count("set-cookie");
+                for (int i=0; i < cookies_count; i++)
+                {
+                    std::string cookie_str = res->get_header_value("set-cookie", i);
+                    std::vector<std::string> cookies = Util::Split(cookie_str, ";");
+                    for (std::vector<std::string>::iterator it = cookies.begin(); it != cookies.end();)
+                    {
+                        std::vector<std::string> cookie = Util::Split(*it, "=");
+                        this->cookies[Util::Trim(cookie[0], " ")] = Util::Trim(cookie[1], " ");
+                    }
+                }
+                return 1;
+            }
+            else
+            {
+                return 0;
+            }
+        }
+        else
+        {
+            return 0;
+        }
+    }
+    else
+    {
+        return 0;
+    }
+}
+
 std::vector<DirEntry> ArchiveOrgClient::ListDir(const std::string &path)
 {
     std::vector<DirEntry> out;
     DirEntry entry;
     Util::SetupPreviousFolder(path, &entry);
     out.push_back(entry);
+    Headers headers;
+    SetCookies(headers);
 
     std::string encoded_path = httplib::detail::encode_url(GetFullPath(path) + "/");
-    if (auto res = client->Get(encoded_path))
+    if (auto res = client->Get(encoded_path, headers))
     {
         lxb_status_t status;
         lxb_dom_attr_t *attr;

source/clients/archiveorg.h

@@ -11,7 +11,11 @@
 class ArchiveOrgClient : public BaseClient
 {
 public:
+    int Connect(const std::string &url, const std::string &username, const std::string &password);
     std::vector<DirEntry> ListDir(const std::string &path);
+
+private:
+    int Login(const std::string &username, const std::string &password);
 };
 
 #endif

source/clients/baseclient.cpp

@@ -20,6 +20,21 @@ BaseClient::~BaseClient()
         delete client;
 };
 
+int BaseClient::SetCookies(Headers &headers)
+{
+    if (this->cookies.size() > 0)
+    {
+        std::string cookie;
+        for (std::map<std::string, std::string>::iterator it = this->cookies.begin(); it != this->cookies.end();)
+        {
+            cookie.append(it->first).append("=").append(it->second).append("; ");
+        }
+        headers["Cookie"] = cookie;
+    }
+
+    return 1;
+}
+
 int BaseClient::Connect(const std::string &url, const std::string &username, const std::string &password)
 {
     this->host_url = url;
@@ -81,7 +96,10 @@ int BaseClient::Rmdir(const std::string &path, bool recursive)
 
 int BaseClient::Size(const std::string &path, int64_t *size)
 {
-    if (auto res = client->Head(GetFullPath(path)))
+    Headers headers;
+    SetCookies(headers);
+
+    if (auto res = client->Head(GetFullPath(path), headers))
     {
         if (HTTP_SUCCESS(res->status))
         {
@@ -128,8 +146,10 @@ int BaseClient::Get(const std::string &outputfile, const std::string &path, uint
     std::ofstream file_stream(outputfile, std::ios::binary);
     bytes_transfered = 0;
     sceRtcGetCurrentTick(&prev_tick);
+    Headers headers;
+    SetCookies(headers);
 
-    if (auto res = client->Get(GetFullPath(path),
+    if (auto res = client->Get(GetFullPath(path), headers,
                                [&](const char *data, size_t data_length)
                                {
                                    file_stream.write(data, data_length);
@@ -149,7 +169,10 @@ int BaseClient::Get(const std::string &outputfile, const std::string &path, uint
 
 int BaseClient::Get(SplitFile *split_file, const std::string &path, uint64_t offset)
 {
-    if (auto res = client->Get(GetFullPath(path),
+    Headers headers;
+    SetCookies(headers);
+
+    if (auto res = client->Get(GetFullPath(path), headers,
                                [&](const char *data, size_t data_length)
                                {
                                    if (!split_file->IsClosed())
@@ -175,6 +198,8 @@ int BaseClient::GetRange(const std::string &path, DataSink &sink, uint64_t size,
     char range_header[64];
     sprintf(range_header, "bytes=%lu-%lu", offset, offset + size - 1);
     Headers headers = {{"Range", range_header}};
+    SetCookies(headers);
+
     size_t bytes_read = 0;
     if (auto res = client->Get(GetFullPath(path), headers,
                                [&](const char *data, size_t data_length)
@@ -198,6 +223,8 @@ int BaseClient::GetRange(const std::string &path, void *buffer, uint64_t size, u
     char range_header[64];
     sprintf(range_header, "bytes=%lu-%lu", offset, offset + size - 1);
     Headers headers = {{"Range", range_header}};
+    SetCookies(headers);
+
     size_t bytes_read = 0;
     std::vector<char> body;
     if (auto res = client->Get(GetFullPath(path), headers,
@@ -257,6 +284,8 @@ int BaseClient::Head(const std::string &path, void *buffer, uint64_t len)
     char range_header[64];
     sprintf(range_header, "bytes=%lu-%lu", 0L, len - 1);
     Headers headers = {{"Range", range_header}};
+    SetCookies(headers);
+
     size_t bytes_read = 0;
     std::vector<char> body;
     if (auto res = client->Get(GetFullPath(path), headers,

source/clients/baseclient.h

@@ -49,11 +49,14 @@ public:
     static std::string UnEscape(const std::string &url);
 
 protected:
+    int SetCookies(httplib::Headers &headers);
+
     httplib::Client *client;
     std::string base_path;
     std::string host_url;
     char response[512];
     bool connected = false;
+    std::map<std::string, std::string> cookies;
 };
 
 #endif