Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@ jobs:
submodules: recursive
- name: Build and test
run: |
sudo apt-get update && sudo apt-get -y install python3 wget zlib1g-dev gcc g++ libcurl4-openssl-dev libxml2 xz-utils lzma build-essential libssl-dev
pip3 install rangehttpserver
sudo apt-get update && sudo apt-get -y install python3 python3-rangehttpserver wget zip zlib1g-dev gcc g++ libcurl4-openssl-dev xz-utils lzma build-essential libssl-dev
make && make run_tests
- uses: actions/upload-artifact@v4
with:
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ build:
g++ -g cli/*.cpp src/*.cpp src/repos/*.cpp -std=gnu++20 -lm -I $(RAPIDJSON) $(MINIZIP_O) -I $(MINIZIP) -lz -lcurl -o get -DAPP_VERSION=\"$(APP_VERSION)\"

run_tests:
rm -rf tests/.get/packages tests/.get/tmp tests/.get/repos
rm -rf tests/.get/packages tests/.get/tmp tests/.get/repos tests/.get-corrupt
gcc -c $(MINIZIP)/*.c
g++ -g tests/*.cpp src/*.cpp src/repos/*.cpp -std=gnu++20 -lm -L/usr/lib -I $(RAPIDJSON) $(MINIZIP_O) -I $(MINIZIP) -lz -lcurl -o get_tests
@echo "Starting test HTTP server with range request support..."
Expand Down
75 changes: 63 additions & 12 deletions src/Get.cpp
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
#include <algorithm>
#include <cstdarg>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <iostream>
#include <sys/stat.h>
#include <sys/types.h>
#include <unordered_set>

#if !defined(WIN32) && !defined(_WIN32)
#include <unistd.h>
#endif

#include "rapidjson/document.h"
#include "rapidjson/istreamwrapper.h"
#include "rapidjson/stringbuffer.h"
Expand All @@ -18,6 +24,26 @@

using namespace rapidjson;

// Writes the whole string in one go and syncs it to the card, so we don't
// leave a half written repos.json behind. No tmp file or rename here, since
// rename doesn't replace an existing file on every homebrew platform.
static bool writeWholeFile(const std::string& path, const char* data)
{
FILE* file = fopen(path.c_str(), "wb");
if (!file)
return false;

size_t len = strlen(data);
bool ok = fwrite(data, 1, len, file) == len && fflush(file) == 0;

#if !defined(WIN32) && !defined(_WIN32)
if (ok)
fsync(fileno(file));
#endif

return fclose(file) == 0 && ok;
}

bool debug = false;

Get::Get(
Expand Down Expand Up @@ -184,13 +210,13 @@ void Get::saveRepos() {

d.AddMember("repos", reposOut, allocator);

std::ofstream file(mRepos_path);
StringBuffer buffer;
Writer<StringBuffer> writer(buffer);
d.Accept(writer);
std::cout << buffer.GetString() << std::endl;
file << buffer.GetString();
file.close();

if (!writeWholeFile(mRepos_path, buffer.GetString()))
printf("--> Could not write %s\n", mRepos_path.c_str());
}

/**
Expand All @@ -201,27 +227,32 @@ void Get::loadRepos()
repos.clear();

auto& config_path = mRepos_path;

auto makeDefaultRepo = [this]() -> std::unique_ptr<Repo> {
#if defined(WII) || defined(_3DS) || defined(WII_MOCK)
return GetRepo::createRepo("Default Repo", this->mDefaultRepo, true, this->mDefaultRepoType, mPkg_path);
#else
return std::make_unique<GetRepo>("Default Repo", this->mDefaultRepo, true);
#endif
};

std::ifstream ifs(config_path);
bool generated = false;

if (!ifs.good() || ifs.peek() == std::ifstream::traits_type::eof())
{
printf("--> Could not load repos from %s, generating default GET repos.json\n", config_path.c_str());

#if defined(WII) || defined(_3DS) || defined(WII_MOCK)
auto defaultRepo = GetRepo::createRepo("Default Repo", this->mDefaultRepo, true, this->mDefaultRepoType, mPkg_path);
#else
auto defaultRepo = std::make_unique<GetRepo>("Default Repo", this->mDefaultRepo, true);
#endif
auto defaultRepo = makeDefaultRepo();

Document d;
d.Parse(Repo::generateRepoJson(*defaultRepo).c_str());

std::ofstream file(config_path);
StringBuffer buffer;
Writer<StringBuffer> writer(buffer);
d.Accept(writer);
file << buffer.GetString();
file.close();
ifs.close();
writeWholeFile(config_path, buffer.GetString());

ifs = std::ifstream(config_path);

Expand All @@ -233,6 +264,7 @@ void Get::loadRepos()
repos.push_back(std::move(defaultRepo));
return;
}
generated = true;
}

IStreamWrapper isw(ifs);
Expand All @@ -242,7 +274,26 @@ void Get::loadRepos()

if (!ok || !doc.HasMember("repos"))
{
printf("--> Invalid format in %s", config_path.c_str());
printf("--> Invalid format in %s\n", config_path.c_str());
ifs.close();

// keep the broken file (e.g. NUL bytes after a power cut) as .bad and
// start over with a default one. The old .bad goes first, since rename
// can't overwrite on some platforms.
if (!generated)
{
std::string badPath = config_path + ".bad";
std::remove(badPath.c_str());
if (std::rename(config_path.c_str(), badPath.c_str()) == 0)
{
printf("--> Moved %s to %s\n", config_path.c_str(), badPath.c_str());
loadRepos();
return;
}
}

// can't move it or our own default is broken, so stay in memory
repos.push_back(makeDefaultRepo());
return;
}

Expand Down
25 changes: 12 additions & 13 deletions src/Utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@
#include <sys/types.h>
#endif

#include <algorithm>
#include <cstdint>
#include <ctime>
#include <dirent.h>
#include <errno.h>
Expand All @@ -36,10 +34,8 @@
#include <iostream>
#include <memory>
#include <sstream>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <sys/stat.h>
#include <unistd.h>
#include <iomanip>
Expand Down Expand Up @@ -153,14 +149,17 @@ char* my_strptime(const char* s,
const char* f,
struct tm* tm)
{
std::istringstream input(s);
input.imbue(std::locale(setlocale(LC_ALL, nullptr)));
input >> std::get_time(tm, f);
if (input.fail())
{
return nullptr;
}
return (char*)(s + input.tellg());
// just use normal strptime, the DIY implementation was being Weird
return strptime(s, f, tm);

// std::istringstream input(s);
// input.imbue(std::locale(setlocale(LC_ALL, nullptr)));
// input >> std::get_time(tm, f);
// if (input.fail())
// {
// return nullptr;
// }
// return (char*)(s + input.tellg());
}

// http://stackoverflow.com/a/11366985
Expand Down Expand Up @@ -762,4 +761,4 @@ std::string getHumanReadableBytes(uint64_t bytes)
ss.precision(places);
ss << std::fixed << size << " " << suffixes[suffix];
return ss.str();
}
}
3 changes: 2 additions & 1 deletion src/Utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#endif

#include <algorithm>
#include <cstdint>
#include <stdio.h>
#include <string>

Expand Down Expand Up @@ -90,4 +91,4 @@ std::string dir_name(const std::string& file_path);
bool compareLen(const std::string& a, const std::string& b);
bool is_dir(std::string_view path, struct dirent* ent);
std::string getHumanReadableBytes(uint64_t bytes);
#endif
#endif
50 changes: 50 additions & 0 deletions tests/15_CorruptReposJson.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#include "tests.hpp"
#include <fstream>

// A repos.json full of NUL bytes (what a FAT card can leave behind after a
// power cut) should be kept as repos.json.bad and replaced by the default.

class CorruptReposJsonTest : public Test {
public:
CorruptReposJsonTest() {
purpose = "Recover from a corrupted repos.json";
}
bool execute()
{
std::string dir = "./tests/.get-corrupt/";
std::string reposPath = dir + "repos.json";
std::string badPath = reposPath + ".bad";

mkpath("./tests/.get-corrupt");
std::remove(badPath.c_str());

{
std::ofstream file(reposPath, std::ios::binary);
file << std::string(200, '\0');
}

Get corrupt(dir, URL "a", false);
auto& repos = corrupt.getRepos();
if (repos.size() != 1 || repos[0]->getUrl() != URL "a")
{
error << "Expected only the default repo, found " << repos.size() << " repos";
return false;
}

if (!exists(badPath.c_str()))
{
error << "The broken file wasn't kept as " << badPath;
return false;
}

// the new file on disk has to load again
Get reloaded(dir, URL "a", false);
if (reloaded.getRepos().size() != 1)
{
error << "The regenerated repos.json didn't load again";
return false;
}

return true;
}
};
2 changes: 2 additions & 0 deletions tests/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "12_RepoAddAndRemove.hpp"
#include "13_ResumeDownloadTest.hpp"
#include "14_LargeDownloadProgressTest.hpp"
#include "15_CorruptReposJson.hpp"


using namespace std;
Expand Down Expand Up @@ -65,6 +66,7 @@ int main()
tests.push_back(std::make_unique<FakeManifestUpgradeTest>());
tests.push_back(std::make_unique<ResumeDownloadTest>());
tests.push_back(std::make_unique<LargeDownloadProgressTest>());
tests.push_back(std::make_unique<CorruptReposJsonTest>());
// tests.push_back(std::make_unique<PackagesSortedTest>());
// tests.push_back(std::make_unique<RepoAddAndRemoveTest>());

Expand Down