diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb19cb5..c6e83c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,8 +17,7 @@ jobs: submodules: recursive - name: Build and test run: | - sudo apt-get update && sudo apt-get -y install python3 wget zlib1g-dev gcc g++ libcurl4-openssl-dev libxml2 xz-utils lzma build-essential libssl-dev - pip3 install rangehttpserver + sudo apt-get update && sudo apt-get -y install python3 python3-rangehttpserver wget zip zlib1g-dev gcc g++ libcurl4-openssl-dev xz-utils lzma build-essential libssl-dev make && make run_tests - uses: actions/upload-artifact@v4 with: diff --git a/Makefile b/Makefile index e6e053c..053db31 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ build: g++ -g cli/*.cpp src/*.cpp src/repos/*.cpp -std=gnu++20 -lm -I $(RAPIDJSON) $(MINIZIP_O) -I $(MINIZIP) -lz -lcurl -o get -DAPP_VERSION=\"$(APP_VERSION)\" run_tests: - rm -rf tests/.get/packages tests/.get/tmp tests/.get/repos + rm -rf tests/.get/packages tests/.get/tmp tests/.get/repos tests/.get-corrupt gcc -c $(MINIZIP)/*.c g++ -g tests/*.cpp src/*.cpp src/repos/*.cpp -std=gnu++20 -lm -L/usr/lib -I $(RAPIDJSON) $(MINIZIP_O) -I $(MINIZIP) -lz -lcurl -o get_tests @echo "Starting test HTTP server with range request support..." diff --git a/src/Get.cpp b/src/Get.cpp index 70e851b..ade1049 100644 --- a/src/Get.cpp +++ b/src/Get.cpp @@ -1,11 +1,17 @@ #include #include +#include +#include #include #include #include #include #include +#if !defined(WIN32) && !defined(_WIN32) +#include +#endif + #include "rapidjson/document.h" #include "rapidjson/istreamwrapper.h" #include "rapidjson/stringbuffer.h" @@ -18,6 +24,26 @@ using namespace rapidjson; +// Writes the whole string in one go and syncs it to the card, so we don't +// leave a half written repos.json behind. No tmp file or rename here, since +// rename doesn't replace an existing file on every homebrew platform. +static bool writeWholeFile(const std::string& path, const char* data) +{ + FILE* file = fopen(path.c_str(), "wb"); + if (!file) + return false; + + size_t len = strlen(data); + bool ok = fwrite(data, 1, len, file) == len && fflush(file) == 0; + +#if !defined(WIN32) && !defined(_WIN32) + if (ok) + fsync(fileno(file)); +#endif + + return fclose(file) == 0 && ok; +} + bool debug = false; Get::Get( @@ -184,13 +210,13 @@ void Get::saveRepos() { d.AddMember("repos", reposOut, allocator); - std::ofstream file(mRepos_path); StringBuffer buffer; Writer writer(buffer); d.Accept(writer); std::cout << buffer.GetString() << std::endl; - file << buffer.GetString(); - file.close(); + + if (!writeWholeFile(mRepos_path, buffer.GetString())) + printf("--> Could not write %s\n", mRepos_path.c_str()); } /** @@ -201,27 +227,32 @@ void Get::loadRepos() repos.clear(); auto& config_path = mRepos_path; + + auto makeDefaultRepo = [this]() -> std::unique_ptr { +#if defined(WII) || defined(_3DS) || defined(WII_MOCK) + return GetRepo::createRepo("Default Repo", this->mDefaultRepo, true, this->mDefaultRepoType, mPkg_path); +#else + return std::make_unique("Default Repo", this->mDefaultRepo, true); +#endif + }; + std::ifstream ifs(config_path); + bool generated = false; if (!ifs.good() || ifs.peek() == std::ifstream::traits_type::eof()) { printf("--> Could not load repos from %s, generating default GET repos.json\n", config_path.c_str()); -#if defined(WII) || defined(_3DS) || defined(WII_MOCK) - auto defaultRepo = GetRepo::createRepo("Default Repo", this->mDefaultRepo, true, this->mDefaultRepoType, mPkg_path); -#else - auto defaultRepo = std::make_unique("Default Repo", this->mDefaultRepo, true); -#endif + auto defaultRepo = makeDefaultRepo(); Document d; d.Parse(Repo::generateRepoJson(*defaultRepo).c_str()); - std::ofstream file(config_path); StringBuffer buffer; Writer writer(buffer); d.Accept(writer); - file << buffer.GetString(); - file.close(); + ifs.close(); + writeWholeFile(config_path, buffer.GetString()); ifs = std::ifstream(config_path); @@ -233,6 +264,7 @@ void Get::loadRepos() repos.push_back(std::move(defaultRepo)); return; } + generated = true; } IStreamWrapper isw(ifs); @@ -242,7 +274,26 @@ void Get::loadRepos() if (!ok || !doc.HasMember("repos")) { - printf("--> Invalid format in %s", config_path.c_str()); + printf("--> Invalid format in %s\n", config_path.c_str()); + ifs.close(); + + // keep the broken file (e.g. NUL bytes after a power cut) as .bad and + // start over with a default one. The old .bad goes first, since rename + // can't overwrite on some platforms. + if (!generated) + { + std::string badPath = config_path + ".bad"; + std::remove(badPath.c_str()); + if (std::rename(config_path.c_str(), badPath.c_str()) == 0) + { + printf("--> Moved %s to %s\n", config_path.c_str(), badPath.c_str()); + loadRepos(); + return; + } + } + + // can't move it or our own default is broken, so stay in memory + repos.push_back(makeDefaultRepo()); return; } diff --git a/tests/15_CorruptReposJson.hpp b/tests/15_CorruptReposJson.hpp new file mode 100644 index 0000000..0f08253 --- /dev/null +++ b/tests/15_CorruptReposJson.hpp @@ -0,0 +1,50 @@ +#include "tests.hpp" +#include + +// A repos.json full of NUL bytes (what a FAT card can leave behind after a +// power cut) should be kept as repos.json.bad and replaced by the default. + +class CorruptReposJsonTest : public Test { + public: + CorruptReposJsonTest() { + purpose = "Recover from a corrupted repos.json"; + } + bool execute() + { + std::string dir = "./tests/.get-corrupt/"; + std::string reposPath = dir + "repos.json"; + std::string badPath = reposPath + ".bad"; + + mkpath("./tests/.get-corrupt"); + std::remove(badPath.c_str()); + + { + std::ofstream file(reposPath, std::ios::binary); + file << std::string(200, '\0'); + } + + Get corrupt(dir, URL "a", false); + auto& repos = corrupt.getRepos(); + if (repos.size() != 1 || repos[0]->getUrl() != URL "a") + { + error << "Expected only the default repo, found " << repos.size() << " repos"; + return false; + } + + if (!exists(badPath.c_str())) + { + error << "The broken file wasn't kept as " << badPath; + return false; + } + + // the new file on disk has to load again + Get reloaded(dir, URL "a", false); + if (reloaded.getRepos().size() != 1) + { + error << "The regenerated repos.json didn't load again"; + return false; + } + + return true; + } +}; diff --git a/tests/main.cpp b/tests/main.cpp index ad6e467..bd6557c 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -16,6 +16,7 @@ #include "12_RepoAddAndRemove.hpp" #include "13_ResumeDownloadTest.hpp" #include "14_LargeDownloadProgressTest.hpp" +#include "15_CorruptReposJson.hpp" using namespace std; @@ -65,6 +66,7 @@ int main() tests.push_back(std::make_unique()); tests.push_back(std::make_unique()); tests.push_back(std::make_unique()); + tests.push_back(std::make_unique()); // tests.push_back(std::make_unique()); // tests.push_back(std::make_unique());