diff options
| -rw-r--r-- | include/erebos/storage.h | 53 | ||||
| -rw-r--r-- | src/main.cpp | 47 | ||||
| -rw-r--r-- | src/storage.cpp | 69 | ||||
| -rw-r--r-- | src/storage.h | 3 | ||||
| -rw-r--r-- | test/storage.test | 75 |
5 files changed, 226 insertions, 21 deletions
diff --git a/include/erebos/storage.h b/include/erebos/storage.h index 24a59df..e894fee 100644 --- a/include/erebos/storage.h +++ b/include/erebos/storage.h @@ -41,6 +41,8 @@ using std::bind; using std::call_once; using std::make_unique; using std::move; +using std::string; +using std::vector; class PartialStorage { @@ -68,6 +70,10 @@ protected: struct Priv; const std::shared_ptr<const Priv> p; PartialStorage(const std::shared_ptr<const Priv> & p): p(p) {} + +public: + // For test usage + const Priv & priv() const { return *p; } }; class Storage : public PartialStorage @@ -200,6 +206,12 @@ public: const Storage & storage() const; + vector<Ref> previous() const; + class Generation generation() const; + +private: + class Generation generationLocked() const; + protected: Ref(const std::shared_ptr<const Priv> p): PartialRef(p) {} }; @@ -349,6 +361,19 @@ std::optional<Stored<T>> RecordT<S>::Item::as() const return std::nullopt; } +class Generation +{ +public: + Generation(); + static Generation next(const vector<Generation> &); + + explicit operator string() const; + +private: + Generation(size_t); + size_t gen; +}; + template<typename T> class Stored { @@ -382,6 +407,8 @@ public: const T & operator*() const { init(); return *p->val; } const T * operator->() const { init(); return p->val.get(); } + Generation generation() const { return p->ref.generation(); } + std::vector<Stored<T>> previous() const; bool precedes(const Stored<T> &) const; @@ -448,27 +475,11 @@ Ref Stored<T>::store(const Storage & st) const template<typename T> std::vector<Stored<T>> Stored<T>::previous() const { - auto rec = p->ref->asRecord(); - if (!rec) - return {}; - - auto sdata = rec->item("SDATA").asRef(); - if (sdata) { - auto drec = sdata.value()->asRecord(); - if (!drec) - return {}; - - std::vector<Stored<T>> res; - for (const Record::Item & i : drec->items("SPREV")) - if (auto x = i.as<T>()) - res.push_back(*x); - return res; - } - - std::vector<Stored<T>> res; - for (const Record::Item & i : rec->items("PREV")) - if (auto x = i.as<T>()) - res.push_back(*x); + auto refs = p->ref.previous(); + vector<Stored<T>> res; + res.reserve(refs.size()); + for (const auto & r : refs) + res.push_back(Stored<T>::load(r)); return res; } diff --git a/src/main.cpp b/src/main.cpp index cb9002e..a3facea 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4,6 +4,8 @@ #include <erebos/storage.h> #include <erebos/sync.h> +#include "storage.h" + #include <arpa/inet.h> #include <netinet/in.h> #include <sys/socket.h> @@ -37,6 +39,7 @@ using std::promise; using std::scoped_lock; using std::string; using std::thread; +using std::to_string; using std::unique_ptr; using std::vector; @@ -104,6 +107,48 @@ struct Command function<void(const vector<string> &)> action; }; +void store(const vector<string> & args) +{ + auto type = args.at(0); + + vector<uint8_t> inner, data; + + char * line = nullptr; + size_t size = 0; + + while (getline(&line, &size, stdin) > 0 && strlen(line) > 1) + copy(line, line + strlen(line), std::back_inserter(inner)); + + free(line); + + auto inserter = std::back_inserter(data); + copy(type.begin(), type.end(), inserter); + inserter = ' '; + + auto slen = to_string(inner.size()); + copy(slen.begin(), slen.end(), inserter); + inserter = '\n'; + + copy(inner.begin(), inner.end(), inserter); + + auto digest = st.priv().storeBytes(data); + + ostringstream ss; + ss << "store-done " << string(digest); + printLine(ss.str()); +} + +void storedGeneration(const vector<string> & args) +{ + auto ref = st.ref(Digest(args.at(0))); + if (!ref) + throw invalid_argument("ref " + args.at(0) + " not found"); + + ostringstream ss; + ss << "stored-generation " << string(ref->digest()) << " " << string(ref->generation()); + printLine(ss.str()); +} + void createIdentity(const vector<string> & args) { optional<Identity> identity; @@ -292,6 +337,8 @@ void attachReject(const vector<string> & params) } vector<Command> commands = { + { "store", store }, + { "stored-generation", storedGeneration }, { "create-identity", createIdentity }, { "start-server", startServer }, { "stop-server", stopServer }, diff --git a/src/storage.cpp b/src/storage.cpp index 05b100a..5784309 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -979,6 +979,56 @@ const Storage & Ref::storage() const return *static_cast<const Storage*>(p->storage.get()); } +vector<Ref> Ref::previous() const +{ + auto rec = (**this).asRecord(); + if (!rec) + return {}; + + vector<Ref> res; + + auto sdata = rec->item("SDATA").asRef(); + if (sdata) { + auto drec = sdata.value()->asRecord(); + if (!drec) + return {}; + + for (const Record::Item & i : drec->items("SPREV")) + if (auto x = i.asRef()) + res.push_back(*x); + return res; + } + + for (const Record::Item & i : rec->items("PREV")) + if (auto x = i.asRef()) + res.push_back(*x); + return res; +} + +Generation Ref::generation() const +{ + scoped_lock lock(p->storage->p->generationCacheLock); + return generationLocked(); +} + +Generation Ref::generationLocked() const +{ + auto it = p->storage->p->generationCache.find(p->digest); + if (it != p->storage->p->generationCache.end()) + return it->second; + + auto prev = previous(); + vector<Generation> pgen; + pgen.reserve(prev.size()); + for (const auto & r : prev) + pgen.push_back(r.generationLocked()); + + auto gen = Generation::next(pgen); + + p->storage->p->generationCache.emplace(p->digest, gen); + return gen; +} + template<class S> RecordT<S>::Item::operator bool() const @@ -1366,6 +1416,25 @@ optional<Blob> ObjectT<S>::asBlob() const template class erebos::ObjectT<Storage>; template class erebos::ObjectT<PartialStorage>; + +Generation::Generation(): Generation(0) {} +Generation::Generation(size_t g): gen(g) {} + +Generation Generation::next(const vector<Generation> & prev) +{ + Generation ret; + for (const auto g : prev) + if (ret.gen <= g.gen) + ret.gen = g.gen + 1; + return ret; +} + +Generation::operator string() const +{ + return to_string(gen); +} + + vector<Stored<Object>> erebos::collectStoredObjects(const Stored<Object> & from) { unordered_set<Digest> seen; diff --git a/src/storage.h b/src/storage.h index 0f7c3bf..ef335b8 100644 --- a/src/storage.h +++ b/src/storage.h @@ -164,6 +164,9 @@ struct PartialStorage::Priv optional<Digest> copy(const typename S::Ref &, vector<Digest> *) const; template<class S> optional<Digest> copy(const ObjectT<S> &, vector<Digest> *) const; + + mutable mutex generationCacheLock {}; + mutable unordered_map<Digest, Generation> generationCache {}; }; struct PartialRef::Priv diff --git a/test/storage.test b/test/storage.test new file mode 100644 index 0000000..2ce87d5 --- /dev/null +++ b/test/storage.test @@ -0,0 +1,75 @@ +test: + spawn on node1 as p1 + + # Diamond history + send to p1: + "store rec" + "text:t First root" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r1 + guard r1 == "blake2#c4a8c69fbc8398acf76a2ec1e5a191f339c4d03c3eb425af19d6d7d5efac6b8e" + + send to p1: + "store rec" + "PREV:r $r1" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r2 + + send to p1: + "store rec" + "text:t Second branch" + "PREV:r $r1" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r3 + + send to p1: + "store rec" + "PREV:r $r2" + "PREV:r $r3" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r4 + + send to p1 "stored-generation $r1" + expect from p1 /stored-generation $r1 0/ + + send to p1 "stored-generation $r2" + expect from p1 /stored-generation $r2 1/ + + send to p1 "stored-generation $r3" + expect from p1 /stored-generation $r3 1/ + + send to p1 "stored-generation $r4" + expect from p1 /stored-generation $r4 2/ + + # Attach second root + send to p1: + "store rec" + "text:t Second root" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r2_1 + + send to p1: + "store rec" + "PREV:r $r2_1" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r2_2 + + send to p1: + "store rec" + "PREV:r $r2_2" + "PREV:r $r4" + "" + expect from p1: + /store-done (blake2#[0-9a-f]*)/ capture r2_3 + + send to p1 "stored-generation $r2_3" + expect from p1 /stored-generation $r2_3 3/ + + send to p1 "stored-generation $r2_2" + expect from p1 /stored-generation $r2_2 1/ |