From 8ac21c24e49bc3702c55d1c796f969f1d1f6128b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Smr=C5=BE?= Date: Sun, 7 Jun 2020 22:23:40 +0200 Subject: Storage: typed heads --- include/erebos/storage.h | 103 +++++++++++++++++++- src/storage.cpp | 238 ++++++++++++++++++++++++++++++++++++++++++++--- src/storage.h | 25 +++++ 3 files changed, 352 insertions(+), 14 deletions(-) diff --git a/include/erebos/storage.h b/include/erebos/storage.h index 09dc481..57304e1 100644 --- a/include/erebos/storage.h +++ b/include/erebos/storage.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,7 @@ typedef ObjectT PartialObject; class Blob; template class Stored; +template class Head; class PartialStorage { @@ -87,11 +89,24 @@ public: template Stored store(const T &) const; + template std::optional> head(UUID id) const; + template std::vector> heads() const; + template Head storeHead(const T &) const; + template Head storeHead(const Stored &) const; + void storeKey(Ref pubref, const std::vector &) const; std::optional> loadKey(Ref pubref) const; protected: + template friend class Head; + Storage(const std::shared_ptr p): PartialStorage(p) {} + + std::optional headRef(UUID type, UUID id) const; + std::vector> headRefs(UUID type) const; + static UUID storeHead(UUID type, const Ref & ref); + static bool replaceHead(UUID type, UUID id, const Ref & old, const Ref & ref); + static std::optional updateHead(UUID type, UUID id, const Ref & old, const std::function &); }; class Digest @@ -316,12 +331,14 @@ class Stored { Stored(Ref ref, std::future && val): mref(ref), mval(std::move(val)) {} friend class Storage; + friend class Head; public: Stored(const Stored &) = default; Stored(Stored &&) = default; Stored & operator=(const Stored &) = default; Stored & operator=(Stored &&) = default; + Stored(const Ref &); static Stored load(const Ref &); Ref store(const Storage &) const; @@ -359,12 +376,18 @@ Stored Storage::store(const T & val) const })); } +template +Stored::Stored(const Ref & ref): + mref(ref), + mval(std::async(std::launch::deferred, [ref] { + return T::load(ref); + })) +{} + template Stored Stored::load(const Ref & ref) { - return Stored(ref, std::async(std::launch::deferred, [ref] { - return T::load(ref); - })); + return Stored(ref); } template @@ -436,6 +459,80 @@ void filterAncestors(std::vector> & xs) } } +template +class Head +{ + Head(UUID id, Ref ref, std::future && val): + mid(id), mstored(ref, std::move(val)) {} + friend class Storage; +public: + Head(UUID id, Ref ref): mid(id), mstored(ref) {} + + const T & operator*() const { return *mstored; } + const T * operator->() const { return &(*mstored); } + + std::vector> previous() const; + bool precedes(const Stored &) const; + + UUID id() const { return mid; } + const Stored & stored() const { return mstored; } + const Ref & ref() const { return mstored.ref(); } + + std::optional> update(const std::function(const Stored &)> &) const; + +private: + UUID mid; + Stored mstored; +}; + +template +std::optional> Storage::head(UUID id) const +{ + if (auto ref = headRef(T::headTypeId, id)) + return Head(id, *ref); + return std::nullopt; +} + +template +std::vector> Storage::heads() const +{ + std::vector> res; + for (const auto & x : headRefs(T::headTypeId)) + res.emplace_back(std::get(x), std::get(x)); + return res; +} + +template +Head Storage::storeHead(const T & val) const +{ + auto ref = val.store(*this); + auto id = storeHead(T::headTypeId, ref); + return Head(id, ref, std::async(std::launch::deferred, [val] { + return val; + })); +} + +template +Head Storage::storeHead(const Stored & val) const +{ + auto id = storeHead(T::headTypeId, val.ref()); + return Head(id, val.ref(), val.mval); +} + +template +std::optional> Head::update(const std::function(const Stored &)> & f) const +{ + auto res = Storage::updateHead(T::headTypeId, mid, ref(), [&f, this](const Ref & r) { + return f(r == ref() ? stored() : Stored::load(r)).ref(); + }); + + if (!res) + return std::nullopt; + if (*res == ref()) + return *this; + return Head(mid, *res); +} + } namespace std diff --git a/src/storage.cpp b/src/storage.cpp index 16662df..b5bff8c 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -18,6 +18,7 @@ using namespace erebos; using std::array; using std::copy; +using std::get; using std::holds_alternative; using std::ifstream; using std::is_same_v; @@ -30,7 +31,6 @@ using std::runtime_error; using std::shared_ptr; using std::string; using std::to_string; -using std::tuple; FilesystemStorage::FilesystemStorage(const fs::path & path): root(path) @@ -125,16 +125,7 @@ void FilesystemStorage::storeBytes(const Digest & digest, const vector auto lock = path; lock += ".lock"; - fs::create_directories(path.parent_path()); - - // No way to use open exclusively in c++ stdlib - FILE *f = nullptr; - for (int i = 0; i < 10; i++) { - f = fopen(lock.c_str(), "wbxe"); - if (f || errno != EEXIST) - break; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } + FILE * f = openLockFile(lock); if (fs::exists(path)) { if (f) { fclose(f); @@ -171,6 +162,81 @@ void FilesystemStorage::storeBytes(const Digest & digest, const vector fs::rename(lock, path); } +optional FilesystemStorage::headRef(UUID type, UUID id) const +{ + ifstream fin(headPath(type, id)); + if (!fin) + return nullopt; + + string sdgst; + fin >> sdgst; + return Digest(sdgst); +} + +vector> FilesystemStorage::headRefs(UUID type) const +{ + vector> res; + string stype(type); + fs::path ptype(stype.begin(), stype.end()); + try { + for (const auto & p : fs::directory_iterator(root/"heads"/ptype)) + if (auto u = UUID::fromString(p.path().filename())) { + ifstream fin(p.path()); + if (fin) { + string sdgst; + fin >> sdgst; + res.emplace_back(*u, Digest(sdgst)); + } + } + } catch (const fs::filesystem_error & e) { + if (e.code() == std::errc::no_such_file_or_directory) + return {}; + throw e; + } + return res; +} + +UUID FilesystemStorage::storeHead(UUID type, const Digest & dgst) +{ + auto id = UUID::generate(); + auto path = headPath(type, id); + fs::create_directories(path.parent_path()); + ofstream fout(path); + if (!fout) + throw runtime_error("failed to open head file"); + + fout << string(dgst) << '\n'; + return id; +} + +bool FilesystemStorage::replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) +{ + auto path = headPath(type, id); + auto lock = path; + lock += ".lock"; + FILE * f = openLockFile(lock); + if (!f) + throw runtime_error(("failed to lock head file " + string(path) + + ": " + string(strerror(errno))).c_str()); + + string scur; + ifstream fin(path); + fin >> scur; + fin.close(); + Digest cur(scur); + + if (cur != old) { + fclose(f); + unlink(lock.c_str()); + return false; + } + + fprintf(f, "%s\n", string(dgst).c_str()); + fclose(f); + fs::rename(lock, path); + return true; +} + optional> FilesystemStorage::loadKey(const Digest & pubref) const { fs::path path = keyPath(pubref); @@ -201,12 +267,37 @@ fs::path FilesystemStorage::objectPath(const Digest & digest) const fs::path(name.begin() + 2, name.end()); } +fs::path FilesystemStorage::headPath(UUID type, UUID id) const +{ + string stype(type), sid(id); + return root/"heads"/ + fs::path(stype.begin(), stype.end())/ + fs::path(sid.begin(), sid.end()); +} + fs::path FilesystemStorage::keyPath(const Digest & digest) const { string name(digest); return root/"keys"/fs::path(name.begin(), name.end()); } +FILE * FilesystemStorage::openLockFile(const fs::path & path) const +{ + fs::create_directories(path.parent_path()); + + // No way to use open exclusively in c++ stdlib + FILE *f = nullptr; + for (int i = 0; i < 10; i++) { + f = fopen(path.c_str(), "wbxe"); + if (f || errno != EEXIST) + break; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + return f; +} + + bool MemoryStorage::contains(const Digest & digest) const { return storage.find(digest) != storage.end(); @@ -225,6 +316,57 @@ void MemoryStorage::storeBytes(const Digest & digest, const vector & co storage.emplace(digest, content); } +optional MemoryStorage::headRef(UUID type, UUID id) const +{ + auto it = heads.find(type); + if (it == heads.end()) + return nullopt; + + for (const auto & x : it->second) + if (get(x) == id) + return get(x); + + return nullopt; +} + +vector> MemoryStorage::headRefs(UUID type) const +{ + auto it = heads.find(type); + if (it != heads.end()) + return it->second; + return {}; +} + +UUID MemoryStorage::storeHead(UUID type, const Digest & dgst) +{ + auto id = UUID::generate(); + auto it = heads.find(type); + if (it == heads.end()) + heads[type] = { { id, dgst } }; + else + it->second.emplace_back(id, dgst); + return id; +} + +bool MemoryStorage::replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) +{ + auto it = heads.find(type); + if (it == heads.end()) + return false; + + for (auto & x : it->second) + if (get(x) == id) { + if (get(x) == old) { + get(x) = dgst; + return true; + } else { + return false; + } + } + + return false; +} + optional> MemoryStorage::loadKey(const Digest & digest) const { auto it = keys.find(digest); @@ -258,6 +400,42 @@ void ChainStorage::storeBytes(const Digest & digest, const vector & con storage->storeBytes(digest, content); } +optional ChainStorage::headRef(UUID type, UUID id) const +{ + if (auto res = storage->headRef(type, id)) + return res; + if (parent) + return parent->headRef(type, id); + return nullopt; +} + +vector> ChainStorage::headRefs(UUID type) const +{ + auto res = storage->headRefs(type); + if (parent) + for (auto x : parent->headRefs(type)) { + bool add = true; + for (const auto & y : res) + if (get(y) == get(x)) { + add = false; + break; + } + if (add) + res.push_back(x); + } + return res; +} + +UUID ChainStorage::storeHead(UUID type, const Digest & dgst) +{ + return storage->storeHead(type, dgst); +} + +bool ChainStorage::replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) +{ + return storage->replaceHead(type, id, old, dgst); +} + optional> ChainStorage::loadKey(const Digest & digest) const { if (auto res = storage->loadKey(digest)) @@ -453,6 +631,44 @@ optional> Storage::loadKey(Ref pubref) const return p->backend->loadKey(pubref.digest()); } +optional Storage::headRef(UUID type, UUID id) const +{ + if (auto dgst = p->backend->headRef(type, id)) + return ref(*dgst); + return nullopt; +} + +vector> Storage::headRefs(UUID type) const +{ + vector> res; + for (auto x : p->backend->headRefs(type)) + if (auto r = ref(get(x))) + res.emplace_back(get(x), *r); + return res; +} + +UUID Storage::storeHead(UUID type, const Ref & ref) +{ + return ref.storage().p->backend->storeHead(type, ref.digest()); +} + +bool Storage::replaceHead(UUID type, UUID id, const Ref & old, const Ref & ref) +{ + return ref.storage().p->backend->replaceHead(type, id, old.digest(), ref.digest()); +} + +optional Storage::updateHead(UUID type, UUID id, const Ref & old, const std::function & f) +{ + Ref r = f(old); + if (replaceHead(type, id, old, r)) + return r; + + if (auto cur = old.storage().headRef(type, id)) + return updateHead(type, id, *cur, f); + else + return nullopt; +} + Digest::Digest(const string & str) { diff --git a/src/storage.h b/src/storage.h index 1635d40..18ac1ad 100644 --- a/src/storage.h +++ b/src/storage.h @@ -13,6 +13,7 @@ using std::shared_ptr; using std::unique_ptr; using std::unordered_map; using std::unordered_set; +using std::tuple; using std::variant; using std::vector; @@ -29,6 +30,11 @@ public: virtual optional> loadBytes(const Digest &) const = 0; virtual void storeBytes(const Digest &, const vector &) = 0; + virtual optional headRef(UUID type, UUID id) const = 0; + virtual vector> headRefs(UUID type) const = 0; + virtual UUID storeHead(UUID type, const Digest & dgst) = 0; + virtual bool replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) = 0; + virtual optional> loadKey(const Digest &) const = 0; virtual void storeKey(const Digest &, const vector &) = 0; }; @@ -44,6 +50,11 @@ public: virtual optional> loadBytes(const Digest &) const override; virtual void storeBytes(const Digest &, const vector &) override; + virtual optional headRef(UUID type, UUID id) const override; + virtual vector> headRefs(UUID type) const override; + virtual UUID storeHead(UUID type, const Digest & dgst) override; + virtual bool replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) override; + virtual optional> loadKey(const Digest &) const override; virtual void storeKey(const Digest &, const vector &) override; @@ -51,8 +62,11 @@ private: static constexpr size_t CHUNK = 16384; fs::path objectPath(const Digest &) const; + fs::path headPath(UUID id, UUID type) const; fs::path keyPath(const Digest &) const; + FILE * openLockFile(const fs::path & path) const; + fs::path root; }; @@ -67,11 +81,17 @@ public: virtual optional> loadBytes(const Digest &) const override; virtual void storeBytes(const Digest &, const vector &) override; + virtual optional headRef(UUID type, UUID id) const override; + virtual vector> headRefs(UUID type) const override; + virtual UUID storeHead(UUID type, const Digest & dgst) override; + virtual bool replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) override; + virtual optional> loadKey(const Digest &) const override; virtual void storeKey(const Digest &, const vector &) override; private: unordered_map> storage; + unordered_map>> heads; unordered_map> keys; }; @@ -89,6 +109,11 @@ public: virtual optional> loadBytes(const Digest &) const override; virtual void storeBytes(const Digest &, const vector &) override; + virtual optional headRef(UUID type, UUID id) const override; + virtual vector> headRefs(UUID type) const override; + virtual UUID storeHead(UUID type, const Digest & dgst) override; + virtual bool replaceHead(UUID type, UUID id, const Digest & old, const Digest & dgst) override; + virtual optional> loadKey(const Digest &) const override; virtual void storeKey(const Digest &, const vector &) override; -- cgit v1.2.3