improve speed of installing from compress file by increase read buffer

CMakeLists.txt

@@ -71,7 +71,7 @@ add_executable(ezremote_client
 
 add_self(ezremote_client)
 
-add_pkg(ezremote_client ${CMAKE_SOURCE_DIR}/data "RMTC00001" "ezRemote Client" "01.14" 32 0)
+add_pkg(ezremote_client ${CMAKE_SOURCE_DIR}/data "RMTC00001" "ezRemote Client" "01.15" 32 0)
 
 target_link_libraries(ezremote_client
   c

source/actions.cpp

@@ -726,22 +726,29 @@ namespace Actions
                 else if (Util::EndsWith(path,".zip") || Util::EndsWith(path,".rar") ||
                         Util::EndsWith(path,".tar.xz") || Util::EndsWith(path,".tar.gz"))
                 {
-                    ArchiveEntry *entry = ZipUtil::GetPackageEntry(it->path, true);
+                    ArchiveEntry *entry = ZipUtil::GetPackageEntry(it->path, remoteclient);
                     if (entry != nullptr)
                     {
-                        ArchivePkgInstallData *install_data = (ArchivePkgInstallData*) malloc(sizeof(ArchivePkgInstallData));
-                        memset(install_data, 0, sizeof(ArchivePkgInstallData));
+                        while (entry != nullptr)
+                        {
+                            ArchivePkgInstallData *install_data = (ArchivePkgInstallData*) malloc(sizeof(ArchivePkgInstallData));
+                            memset(install_data, 0, sizeof(ArchivePkgInstallData));
 
-                        std::string install_pkg_path = std::string(temp_folder) + "/" + entry->filename;
-                        SplitFile *sp = new SplitFile(install_pkg_path, INSTALL_ARCHIVE_PKG_SPLIT_SIZE);
-                        
-                        install_data->archive_entry = entry;
-                        install_data->split_file = sp;
-                        install_data->stop_write_thread = false;
+                            std::string install_pkg_path = std::string(temp_folder) + "/" + entry->filename;
+                            SplitFile *sp = new SplitFile(install_pkg_path, INSTALL_ARCHIVE_PKG_SPLIT_SIZE);
+                            
+                            install_data->archive_entry = entry;
+                            install_data->split_file = sp;
+                            install_data->stop_write_thread = false;
 
-                        int res = pthread_create(&install_data->thread, NULL, ExtractArchivePkg, install_data);
+                            int res = pthread_create(&install_data->thread, NULL, ExtractArchivePkg, install_data);
 
-                        INSTALLER::InstallArchivePkg(entry->filename, install_data);
+                            INSTALLER::InstallArchivePkg(entry->filename, install_data);
+
+                            ArchiveEntry *previos = entry;
+                            entry = ZipUtil::GetNextPackageEntry(entry);
+                            free(previos);
+                        }
                         success++;
                     }
                     else
@@ -779,7 +786,7 @@ namespace Actions
     void *ExtractArchivePkg(void *argp)
     {
         ssize_t len;
-        char buffer[ARCHIVE_TRANSFER_SIZE];
+        char *buffer = (char*) malloc(ARCHIVE_TRANSFER_SIZE);
 
         ArchivePkgInstallData *install_data = (ArchivePkgInstallData*) argp;
         SplitFile *sp = install_data->split_file;
@@ -788,7 +795,7 @@ namespace Actions
         sp->Open();
         while (!install_data->stop_write_thread)
         {
-            len = archive_read_data(install_data->archive_entry->archive, buffer, sizeof buffer);
+            len = archive_read_data(install_data->archive_entry->archive, buffer, ARCHIVE_TRANSFER_SIZE);
 
             if (len == 0)
                 break;
@@ -807,7 +814,7 @@ namespace Actions
         }
 
         sp->Close();
-
+        free(buffer);
         return NULL;
     }
 
@@ -871,19 +878,26 @@ namespace Actions
                     ArchiveEntry *entry = ZipUtil::GetPackageEntry(it->path);
                     if (entry != nullptr)
                     {
-                        ArchivePkgInstallData *install_data = (ArchivePkgInstallData*) malloc(sizeof(ArchivePkgInstallData));
-                        memset(install_data, 0, sizeof(ArchivePkgInstallData));
+                        while (entry != nullptr)
+                        {
+                            ArchivePkgInstallData *install_data = (ArchivePkgInstallData*) malloc(sizeof(ArchivePkgInstallData));
+                            memset(install_data, 0, sizeof(ArchivePkgInstallData));
 
-                        std::string install_pkg_path = std::string(temp_folder) + "/" + entry->filename;
-                        SplitFile *sp = new SplitFile(install_pkg_path, INSTALL_ARCHIVE_PKG_SPLIT_SIZE);
-                        
-                        install_data->archive_entry = entry;
-                        install_data->split_file = sp;
-                        install_data->stop_write_thread = false;
+                            std::string install_pkg_path = std::string(temp_folder) + "/" + entry->filename;
+                            SplitFile *sp = new SplitFile(install_pkg_path, INSTALL_ARCHIVE_PKG_SPLIT_SIZE);
+                            
+                            install_data->archive_entry = entry;
+                            install_data->split_file = sp;
+                            install_data->stop_write_thread = false;
 
-                        int res = pthread_create(&install_data->thread, NULL, ExtractArchivePkg, install_data);
+                            int res = pthread_create(&install_data->thread, NULL, ExtractArchivePkg, install_data);
 
-                        INSTALLER::InstallArchivePkg(entry->filename, install_data);
+                            INSTALLER::InstallArchivePkg(entry->filename, install_data);
+
+                            ArchiveEntry *previous = entry;
+                            entry = ZipUtil::GetNextPackageEntry(entry);
+                            free(previous);
+                        }
                         success++;
                     }
                     else
@@ -975,7 +989,7 @@ namespace Actions
                 break;
             if (!it->isDir)
             {
-                int ret = ZipUtil::Extract(*it, extract_zip_folder, true);
+                int ret = ZipUtil::Extract(*it, extract_zip_folder, remoteclient);
                 if (ret == 0)
                 {
                     sprintf(status_message, "%s %s", lang_strings[STR_FAILED_TO_EXTRACT], it->name);

source/clients/baseclient.cpp

@@ -280,7 +280,7 @@ ClientType BaseClient::clientType()
 
 uint32_t BaseClient::SupportedActions()
 {
-    return REMOTE_ACTION_DOWNLOAD | REMOTE_ACTION_INSTALL;
+    return REMOTE_ACTION_DOWNLOAD | REMOTE_ACTION_INSTALL | REMOTE_ACTION_EXTRACT;
 }
 
 std::string BaseClient::EncodeUrl(const std::string &url)

source/clients/rclone.cpp

@@ -8,7 +8,6 @@
 #include "util.h"
 #include "system.h"
 #include "windows.h"
-#include "dbglogger.h"
 
 using httplib::Client;
 using httplib::Headers;
@@ -161,7 +160,7 @@ std::vector<DirEntry> RCloneClient::ListDir(const std::string &path)
                 lxb_dom_node_t *size_node = NextChildElement(td_element);
                 value = lxb_dom_node_text_content(size_node->first_child, &value_len);
                 tmp_string = std::string((const char *)value, value_len);
-                entry.file_size = atoi(tmp_string.c_str());
+                entry.file_size = atol(tmp_string.c_str());
                 DirEntry::SetDisplaySize(&entry);
             }
 

source/imgui/imgui.cpp

@@ -10692,7 +10692,7 @@ void ImGui::NavUpdateCreateMoveRequest()
     // When using gamepad, we project the reference nav bounding box into window visible area.
     // This is to allow resuming navigation inside the visible area after doing a large amount of scrolling, since with gamepad all movements are relative
     // (can't focus a visible object like we can with the mouse).
-    if (g.NavMoveSubmitted && g.NavInputSource == ImGuiInputSource_Gamepad && g.NavLayer == ImGuiNavLayer_Main && window != NULL)// && (g.NavMoveFlags & ImGuiNavMoveFlags_Forwarded))
+    /*if (g.NavMoveSubmitted && g.NavInputSource == ImGuiInputSource_Gamepad && g.NavLayer == ImGuiNavLayer_Main && window != NULL)// && (g.NavMoveFlags & ImGuiNavMoveFlags_Forwarded))
     {
         bool clamp_x = (g.NavMoveFlags & (ImGuiNavMoveFlags_LoopX | ImGuiNavMoveFlags_WrapX)) == 0;
         bool clamp_y = (g.NavMoveFlags & (ImGuiNavMoveFlags_LoopY | ImGuiNavMoveFlags_WrapY)) == 0;
@@ -10709,7 +10709,7 @@ void ImGui::NavUpdateCreateMoveRequest()
             window->NavRectRel[g.NavLayer].ClipWithFull(inner_rect_rel);
             g.NavId = g.NavFocusScopeId = 0;
         }
-    }
+    }*/
 
     // For scoring we use a single segment on the left side our current item bounding box (not touching the edge to avoid box overlap with zero-spaced items)
     ImRect scoring_rect;

source/installer.cpp

@@ -325,7 +325,9 @@ namespace INSTALLER
 				{
 					ret = sceAppInstUtilAppUnInstall(cid.c_str());
 					if (ret != 0)
+					{
 						goto err;
+					}
 					goto retry;
 				}
 			}
@@ -338,8 +340,9 @@ namespace INSTALLER
 			}
 		}
 		else if (ret > 0)
+		{
 			goto err;
-
+		}
 		ret = sceBgftServiceDownloadStartTask(task_id);
 		if (ret)
 		{
@@ -855,10 +858,10 @@ namespace INSTALLER
 		pkg_data->stop_write_thread = true;
 		pthread_join(pkg_data->thread, NULL);
 		delete(pkg_data->split_file);
-		free(pkg_data->archive_entry);
 		free(pkg_data);
 		RemoveArchivePkgInstallData(hash);
 		return ret;
 
 	}
+
 }

source/installer.h

@@ -129,6 +129,8 @@ struct ArchivePkgInstallData
     bool stop_write_thread;
 };
 
+static pthread_t bk_install_thid;
+
 namespace INSTALLER
 {
     int Init(void);

source/main.cpp

@@ -11,6 +11,7 @@
 #include <orbis/Pad.h>
 #include <orbis/AudioOut.h>
 #include <orbis/Net.h>
+// #include <dbglogger.h>
 
 #include "imgui.h"
 #include "SDL2/SDL.h"

source/server/http_server.cpp

@@ -1152,7 +1152,6 @@ namespace HttpServer
                 return;
             }
 
-            FileHost::AddCacheDownloadUrl(hash, download_url);
             delete(filehost);
 
 			size_t scheme_pos = download_url.find("://");
@@ -1164,17 +1163,53 @@ namespace HttpServer
             BaseClient *baseclient = new BaseClient();
             baseclient->Connect(host, "", "");
             baseclient->Head(path, &header, sizeof(pkg_header));
-            std::string title = INSTALLER::GetRemotePkgTitle(baseclient, path, &header);
-            delete(baseclient);
 
-            std::string remote_install_url = std::string("http://localhost:") + std::to_string(http_server_port) + "/rmt_inst/Site%2099/" + hash;
-            int rc = INSTALLER::InstallRemotePkg(remote_install_url, &header, title);
-            if (rc == 0)
-            {
-                failed(res, 200, lang_strings[STR_FAIL_INSTALL_FROM_URL_MSG]);
-                return;
+            if (BE32(header.pkg_magic) == 0x7F434E54)
+            {            
+                FileHost::AddCacheDownloadUrl(hash, download_url);
+                std::string title = INSTALLER::GetRemotePkgTitle(baseclient, path, &header);
+                delete(baseclient);
+
+                std::string remote_install_url = std::string("http://localhost:") + std::to_string(http_server_port) + "/rmt_inst/Site%2099/" + hash;
+                int rc = INSTALLER::InstallRemotePkg(remote_install_url, &header, title);
+                if (rc == 0)
+                {
+                    failed(res, 200, lang_strings[STR_FAIL_INSTALL_FROM_URL_MSG]);
+                    return;
+                }
+            }
+            else
+            {
+                ArchiveEntry *entry = ZipUtil::GetPackageEntry(path, baseclient);
+                if (entry != nullptr)
+                {
+                    ArchivePkgInstallData *install_data = (ArchivePkgInstallData*) malloc(sizeof(ArchivePkgInstallData));
+                    memset(install_data, 0, sizeof(ArchivePkgInstallData));
+
+                    std::string install_pkg_path = std::string(temp_folder) + "/" + entry->filename;
+                    SplitFile *sp = new SplitFile(install_pkg_path, INSTALL_ARCHIVE_PKG_SPLIT_SIZE);
+                    
+                    install_data->archive_entry = entry;
+                    install_data->split_file = sp;
+                    install_data->stop_write_thread = false;
+
+                    int ret = pthread_create(&install_data->thread, NULL, Actions::ExtractArchivePkg, install_data);
+
+                    ret = INSTALLER::InstallArchivePkg(entry->filename, install_data);
+                    free(entry);
+
+                    if (ret == 0)
+                    {
+                        failed(res, 200, lang_strings[STR_FAIL_INSTALL_FROM_URL_MSG]);
+                        return;
+                    }
+                }
+                else
+                {
+                    failed(res, 200, lang_strings[STR_FAIL_INSTALL_FROM_URL_MSG]);
+                    return;
+                }
             }
-            
             success(res);
         });
 

source/zip_util.cpp

@@ -288,29 +288,35 @@ namespace ZipUtil
     static int extract2fd(struct archive *a, const std::string &pathname, int fd)
     {
         ssize_t len;
-        unsigned char buffer[ARCHIVE_TRANSFER_SIZE];
+        unsigned char *buffer = (unsigned char *) malloc(ARCHIVE_TRANSFER_SIZE);
 
         /* loop over file contents and write to fd */
         for (int n = 0;; n++)
         {
-            len = archive_read_data(a, buffer, sizeof buffer);
+            len = archive_read_data(a, buffer, ARCHIVE_TRANSFER_SIZE);
 
             if (len == 0)
+            {
+                free(buffer);
                 return 1;
+            }
 
             if (len < 0)
             {
                 sprintf(status_message, "error archive_read_data('%s')", pathname.c_str());
+                free(buffer);
                 return 0;
             }
 
             if (write(fd, buffer, len) != len)
             {
                 sprintf(status_message, "error write('%s')", pathname.c_str());
+                free(buffer);
                 return 0;
             }
         }
 
+        free(buffer);
         return 1;
     }
 
@@ -431,7 +437,7 @@ namespace ZipUtil
         return password;
     }
 
-    static RemoteArchiveData *OpenRemoteArchive(const std::string &file)
+    static RemoteArchiveData *OpenRemoteArchive(const std::string &file, RemoteClient *client)
     {
         RemoteArchiveData *data;
 
@@ -439,8 +445,8 @@ namespace ZipUtil
         memset(data, 0, sizeof(RemoteArchiveData));
 
         data->offset = 0;
-        remoteclient->Size(file, &data->size);
-        data->client = remoteclient;
+        client->Size(file, &data->size);
+        data->client = client;
         data->path = file;
         return data;
     }
@@ -478,7 +484,7 @@ namespace ZipUtil
      * Main loop: open the zipfile, iterate over its contents and decide what
      * to do with each entry.
      */
-    int Extract(const DirEntry &file, const std::string &basepath, bool is_remote)
+    int Extract(const DirEntry &file, const std::string &basepath, RemoteClient *client)
     {
         struct archive *a;
         struct archive_entry *e;
@@ -493,7 +499,7 @@ namespace ZipUtil
         archive_read_support_filter_all(a);
         archive_read_set_passphrase_callback(a, NULL, &passphrase_callback);
 
-        if (!is_remote)
+        if (client == nullptr)
         {
             ret = archive_read_open_filename(a, file.path, ARCHIVE_TRANSFER_SIZE);
             if (ret < ARCHIVE_OK)
@@ -504,7 +510,7 @@ namespace ZipUtil
         }
         else
         {
-            client_data = OpenRemoteArchive(file.path);
+            client_data = OpenRemoteArchive(file.path, client);
             if (client_data == nullptr)
             {
                 sprintf(status_message, "%s", "archive_read_open_filename failed");
@@ -548,7 +554,7 @@ namespace ZipUtil
         return 1;
     }
 
-    ArchiveEntry *GetPackageEntry(const std::string &zip_file, bool is_remote)
+    ArchiveEntry *GetPackageEntry(const std::string &zip_file, RemoteClient *client)
     {
         struct archive *a;
         struct archive_entry *e;
@@ -565,7 +571,7 @@ namespace ZipUtil
         archive_read_support_filter_all(a);
         archive_read_set_passphrase_callback(a, NULL, &passphrase_callback);
 
-        if (!is_remote)
+        if (client == nullptr)
         {
             ret = archive_read_open_filename(a, zip_file.c_str(), ARCHIVE_TRANSFER_SIZE);
             if (ret < ARCHIVE_OK)
@@ -576,7 +582,7 @@ namespace ZipUtil
         }
         else
         {
-            client_data = OpenRemoteArchive(zip_file);
+            client_data = OpenRemoteArchive(zip_file, client);
             if (client_data == nullptr)
             {
                 sprintf(status_message, "%s", "archive_read_open_filename failed");
@@ -613,7 +619,83 @@ namespace ZipUtil
             if (ret == ARCHIVE_EOF)
                 break;
 
-            char *p, *q;
+            if ((pathname = pathdup(archive_entry_pathname(e))) == NULL)
+            {
+                archive_read_data_skip(a);
+                continue;
+            }
+
+            filetype = archive_entry_filetype(e);
+
+            /* sanity checks */
+            if (pathname[0] == '/' ||
+                strncmp(pathname, "../", 3) == 0 ||
+                strstr(pathname, "/../") != NULL)
+            {
+                archive_read_data_skip(a);
+                free(pathname);
+                continue;
+                ;
+            }
+
+            /* I don't think this can happen in a zipfile.. */
+            if (!S_ISREG(filetype))
+            {
+                archive_read_data_skip(a);
+                free(pathname);
+                continue;
+            }
+
+            if (Util::EndsWith(Util::ToLower(pathname), ".pkg"))
+            {
+                pkg_entry = (ArchiveEntry *)malloc(sizeof(ArchiveEntry));
+                memset(pkg_entry, 0, sizeof(ArchiveEntry));
+
+                pkg_entry->archive = a;
+                pkg_entry->entry = e;
+                pkg_entry->client_data = client_data;
+                pkg_entry->filename = pathname;
+                pkg_entry->filesize = archive_entry_size(e);
+
+                free(pathname);
+                return pkg_entry;
+            }
+
+            free(pathname);
+        }
+
+        archive_read_free(a);
+
+        return nullptr;
+    }
+
+    ArchiveEntry *GetNextPackageEntry(ArchiveEntry *archive_entry)
+    {
+        struct archive *a = archive_entry->archive;
+        struct archive_entry *e = nullptr;
+        RemoteArchiveData *client_data = archive_entry->client_data;
+        char *pathname;
+        mode_t filetype;
+        ArchiveEntry *pkg_entry = nullptr;
+        int ret;
+
+        for (;;)
+        {
+            ret = archive_read_next_header(a, &e);
+
+            if (ret < ARCHIVE_OK)
+            {
+                sprintf(status_message, "%s", "archive_read_next_header failed");
+                if (client_data != nullptr)
+                {
+                    free(client_data);
+                }
+                archive_read_free(a);
+                return nullptr;
+            }
+
+            if (ret == ARCHIVE_EOF)
+                break;
 
             if ((pathname = pathdup(archive_entry_pathname(e))) == NULL)
             {

source/zip_util.h

@@ -10,7 +10,7 @@
 #include "common.h"
 #include "fs.h"
 
-#define ARCHIVE_TRANSFER_SIZE (512 * 1024)
+#define ARCHIVE_TRANSFER_SIZE 5242880
 
 static uint8_t MAGIC_ZIP_1[4] = {0x50, 0x4B, 0x03, 0x04};
 static uint8_t MAGIC_ZIP_2[4] = {0x50, 0x4B, 0x05, 0x06};
@@ -49,7 +49,8 @@ struct ArchiveEntry
 namespace ZipUtil
 {
     int ZipAddPath(zipFile zf, const std::string &path, int filename_start, int level);
-    int Extract(const DirEntry &file, const std::string &dir, bool is_remote = false);
-    ArchiveEntry *GetPackageEntry(const std::string &zip_file, bool is_remote = false);
+    int Extract(const DirEntry &file, const std::string &dir, RemoteClient *client = nullptr);
+    ArchiveEntry *GetPackageEntry(const std::string &zip_file, RemoteClient *client = nullptr);
+    ArchiveEntry *GetNextPackageEntry(ArchiveEntry *archive_entry);
 }
 #endif