From 2059c09154870d704d26d10f0462db1e9b4ea676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Smr=C5=BE?= Date: Sat, 4 Jan 2020 21:07:33 +0100 Subject: Partial and memory-backed storage --- src/storage.cpp | 430 ++++++++++++++++++++++++++++++++++++++++---------------- src/storage.h | 95 +++++++++++-- 2 files changed, 395 insertions(+), 130 deletions(-) (limited to 'src') diff --git a/src/storage.cpp b/src/storage.cpp index 0d9b52e..2b4e027 100644 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -21,7 +21,9 @@ using std::array; using std::copy; using std::holds_alternative; using std::ifstream; +using std::is_same_v; using std::make_shared; +using std::make_unique; using std::monostate; using std::nullopt; using std::ofstream; @@ -31,7 +33,8 @@ using std::string; using std::to_string; using std::tuple; -optional Storage::open(fs::path path) +FilesystemStorage::FilesystemStorage(const fs::path & path): + root(path) { if (!fs::is_directory(path)) fs::create_directory(path); @@ -41,42 +44,16 @@ optional Storage::open(fs::path path) if (!fs::is_directory(path/"heads")) fs::create_directory(path/"heads"); - - return Storage(shared_ptr(new Priv { path })); -} - -bool Storage::operator==(const Storage & other) const -{ - return p == other.p; -} - -bool Storage::operator!=(const Storage & other) const -{ - return p != other.p; -} - -fs::path Storage::Priv::objectPath(const Digest & digest) const -{ - string name(digest); - return root/"objects"/ - fs::path(name.begin(), name.begin() + 2)/ - fs::path(name.begin() + 2, name.end()); -} - -fs::path Storage::Priv::keyPath(const Digest & digest) const -{ - string name(digest); - return root/"keys"/fs::path(name.begin(), name.end()); } -optional Storage::ref(const Digest & digest) const +bool FilesystemStorage::contains(const Digest & digest) const { - return Ref::create(*this, digest); + return fs::exists(objectPath(digest)); } -optional> Storage::Priv::loadBytes(const Digest & digest) const +optional> FilesystemStorage::loadBytes(const Digest & digest) const { - vector in(Priv::CHUNK); + vector in(CHUNK); vector out; size_t decoded = 0; @@ -133,25 +110,9 @@ optional> Storage::Priv::loadBytes(const Digest & digest) const return out; } -optional Storage::loadObject(const Digest & digest) const -{ - auto ocontent = p->loadBytes(digest); - if (!ocontent.has_value()) - return nullopt; - auto content = ocontent.value(); - - array arr; - int ret = blake2b(arr.data(), content.data(), nullptr, - Digest::size, content.size(), 0); - if (ret != 0 || digest != Digest(arr)) - throw runtime_error("digest verification failed"); - - return Object::decode(*this, content); -} - -void Storage::Priv::storeBytes(const Digest & digest, const vector & in) const +void FilesystemStorage::storeBytes(const Digest & digest, const vector & in) { - vector out(Priv::CHUNK); + vector out(CHUNK); z_stream strm; strm.zalloc = Z_NULL; @@ -211,6 +172,180 @@ void Storage::Priv::storeBytes(const Digest & digest, const vector & in fs::rename(lock, path); } +optional> FilesystemStorage::loadKey(const Digest & pubref) const +{ + fs::path path = keyPath(pubref); + std::error_code err; + size_t size = fs::file_size(path, err); + if (err) + return nullopt; + + vector key(size); + ifstream file(keyPath(pubref)); + file.read((char *) key.data(), size); + return key; +} + +void FilesystemStorage::storeKey(const Digest & pubref, const vector & key) +{ + fs::path path = keyPath(pubref); + fs::create_directories(path.parent_path()); + ofstream file(path); + file.write((const char *) key.data(), key.size()); +} + +fs::path FilesystemStorage::objectPath(const Digest & digest) const +{ + string name(digest); + return root/"objects"/ + fs::path(name.begin(), name.begin() + 2)/ + fs::path(name.begin() + 2, name.end()); +} + +fs::path FilesystemStorage::keyPath(const Digest & digest) const +{ + string name(digest); + return root/"keys"/fs::path(name.begin(), name.end()); +} + +bool MemoryStorage::contains(const Digest & digest) const +{ + return storage.find(digest) != storage.end(); +} + +optional> MemoryStorage::loadBytes(const Digest & digest) const +{ + auto it = storage.find(digest); + if (it != storage.end()) + return it->second; + return nullopt; +} + +void MemoryStorage::storeBytes(const Digest & digest, const vector & content) +{ + storage.emplace(digest, content); +} + +optional> MemoryStorage::loadKey(const Digest & digest) const +{ + auto it = keys.find(digest); + if (it != keys.end()) + return it->second; + return nullopt; +} + +void MemoryStorage::storeKey(const Digest & digest, const vector & content) +{ + keys.emplace(digest, content); +} + +bool ChainStorage::contains(const Digest & digest) const +{ + return storage->contains(digest) || + (parent && parent->contains(digest)); +} + +optional> ChainStorage::loadBytes(const Digest & digest) const +{ + if (auto res = storage->loadBytes(digest)) + return res; + if (parent) + return parent->loadBytes(digest); + return nullopt; +} + +void ChainStorage::storeBytes(const Digest & digest, const vector & content) +{ + storage->storeBytes(digest, content); +} + +optional> ChainStorage::loadKey(const Digest & digest) const +{ + if (auto res = storage->loadKey(digest)) + return res; + if (parent) + return parent->loadKey(digest); + return nullopt; +} + +void ChainStorage::storeKey(const Digest & digest, const vector & content) +{ + storage->storeKey(digest, content); +} + + +Storage::Storage(const fs::path & path): + PartialStorage(shared_ptr(new Priv { .backend = make_shared(path) })) +{} + +Storage Storage::deriveEphemeralStorage() const +{ + return Storage(shared_ptr(new Priv { .backend = + make_shared( + make_shared(), + make_unique(p->backend) + )})); +} + +PartialStorage Storage::derivePartialStorage() const +{ + return PartialStorage(shared_ptr(new Priv { .backend = + make_shared( + make_shared(), + make_unique(p->backend) + )})); +} + +bool PartialStorage::operator==(const PartialStorage & other) const +{ + return p == other.p; +} + +bool PartialStorage::operator!=(const PartialStorage & other) const +{ + return p != other.p; +} + +PartialRef PartialStorage::ref(const Digest & digest) const +{ + return PartialRef::create(*this, digest); +} + +optional Storage::ref(const Digest & digest) const +{ + return Ref::create(*this, digest); +} + +optional> PartialStorage::Priv::loadBytes(const Digest & digest) const +{ + auto ocontent = backend->loadBytes(digest); + if (!ocontent.has_value()) + return nullopt; + auto content = ocontent.value(); + + array arr; + int ret = blake2b(arr.data(), content.data(), nullptr, + Digest::size, content.size(), 0); + if (ret != 0 || digest != Digest(arr)) + throw runtime_error("digest verification failed"); + + return content; +} + +optional PartialStorage::loadObject(const Digest & digest) const +{ + if (auto content = p->loadBytes(digest)) + return PartialObject::decode(*this, *content); + return nullopt; +} + +optional Storage::loadObject(const Digest & digest) const +{ + if (auto content = p->loadBytes(digest)) + return Object::decode(*this, *content); + return nullopt; +} + Ref Storage::storeObject(const Object & object) const { // TODO: ensure storage transitively @@ -223,34 +358,24 @@ Ref Storage::storeObject(const Object & object) const throw runtime_error("failed to compute digest"); Digest digest(arr); - p->storeBytes(digest, content); + p->backend->storeBytes(digest, content); return Ref::create(*this, digest).value(); } -Ref Storage::storeObject(const class Record & val) const +Ref Storage::storeObject(const Record & val) const { return storeObject(Object(val)); } -Ref Storage::storeObject(const class Blob & val) const +Ref Storage::storeObject(const Blob & val) const { return storeObject(Object(val)); } void Storage::storeKey(Ref pubref, const vector & key) const { - ofstream file(p->keyPath(pubref.digest())); - file.write((const char *) key.data(), key.size()); + p->backend->storeKey(pubref.digest(), key); } optional> Storage::loadKey(Ref pubref) const { - fs::path path = p->keyPath(pubref.digest()); - std::error_code err; - size_t size = fs::file_size(path, err); - if (err) - return nullopt; - - vector key(size); - ifstream file(p->keyPath(pubref.digest())); - file.read((char *) key.data(), size); - return key; + return p->backend->loadKey(pubref.digest()); } @@ -276,92 +401,125 @@ Digest::operator string() const } -optional Ref::create(Storage st, const Digest & digest) +PartialRef PartialRef::create(PartialStorage st, const Digest & digest) { - if (!fs::exists(st.p->objectPath(digest))) - return nullopt; - auto p = new Priv { - .storage = st, + .storage = make_unique(st), .digest = digest, - .object = {}, }; - p->object = std::async(std::launch::deferred, [p] { - auto obj = p->storage.loadObject(p->digest); - if (!obj.has_value()) - throw runtime_error("failed to decode bytes"); + return PartialRef(shared_ptr(p)); +} - return obj.value(); - }); +const Digest & PartialRef::digest() const +{ + return p->digest; +} - return Ref(shared_ptr(p)); +PartialRef::operator bool() const +{ + return storage().p->backend->contains(p->digest); } -const Digest & Ref::digest() const +const PartialObject PartialRef::operator*() const { - return p->digest; + if (auto res = p->storage->loadObject(p->digest)) + return *res; + throw runtime_error("failed to load object from partial storage"); } -const Object & Ref::operator*() const +unique_ptr PartialRef::operator->() const { - return p->object.get(); + return make_unique(**this); } -const Object * Ref::operator->() const +const PartialStorage & PartialRef::storage() const { - return &p->object.get(); + return *p->storage; +} + +optional Ref::create(Storage st, const Digest & digest) +{ + if (!st.p->backend->contains(digest)) + return nullopt; + + auto p = new Priv { + .storage = make_unique(st), + .digest = digest, + }; + + return Ref(shared_ptr(p)); +} + +const Object Ref::operator*() const +{ + if (auto res = static_cast(p->storage.get())->loadObject(p->digest)) + return *res; + throw runtime_error("falied to load object - corrupted storage"); +} + +unique_ptr Ref::operator->() const +{ + return make_unique(**this); } const Storage & Ref::storage() const { - return p->storage; + return *static_cast(p->storage.get()); } -Record::Item::operator bool() const +template +RecordT::Item::operator bool() const { return !holds_alternative(value); } -optional Record::Item::asInteger() const +template +optional RecordT::Item::asInteger() const { if (holds_alternative(value)) return std::get(value); return nullopt; } -optional Record::Item::asText() const +template +optional RecordT::Item::asText() const { if (holds_alternative(value)) return std::get(value); return nullopt; } -optional> Record::Item::asBinary() const +template +optional> RecordT::Item::asBinary() const { if (holds_alternative>(value)) return std::get>(value); return nullopt; } -optional Record::Item::asRef() const +template +optional RecordT::Item::asRef() const { - if (holds_alternative(value)) - return std::get(value); + if (holds_alternative(value)) + return std::get(value); return nullopt; } -Record::Record(const vector & from): +template +RecordT::RecordT(const vector & from): ptr(new vector(from)) {} -Record::Record(vector && from): +template +RecordT::RecordT(vector && from): ptr(new vector(std::move(from))) {} -Record Record::decode(Storage st, +template +optional> RecordT::decode(const S & st, vector::const_iterator begin, vector::const_iterator end) { @@ -390,28 +548,38 @@ Record Record::decode(Storage st, items->emplace_back(name, value); else if (type == "b") items->emplace_back(name, base64::decode(value)); - else if (type == "r.b2") - items->emplace_back(name, Ref::create(st, Digest(value)).value()); - else + else if (type == "r.b2") { + if constexpr (is_same_v) { + if (auto ref = st.ref(Digest(value))) + items->emplace_back(name, ref.value()); + else + return nullopt; + } else if constexpr (std::is_same_v) { + items->emplace_back(name, st.ref(Digest(value))); + } + } else throw runtime_error("unknown record item type"); begin = newline + 1; } - return Record(items); + return RecordT(items); } -vector Record::encode() const +template +vector RecordT::encode() const { - return Object(*this).encode(); + return ObjectT(*this).encode(); } -const vector & Record::items() const +template +const vector::Item> & RecordT::items() const { return *ptr; } -Record::Item Record::item(const string & name) const +template +typename RecordT::Item RecordT::item(const string & name) const { for (auto item : *ptr) { if (item.name == name) @@ -420,12 +588,14 @@ Record::Item Record::item(const string & name) const return Item("", monostate()); } -Record::Item Record::operator[](const string & name) const +template +typename RecordT::Item RecordT::operator[](const string & name) const { return item(name); } -vector Record::items(const string & name) const +template +vector::Item> RecordT::items(const string & name) const { vector res; for (auto item : *ptr) { @@ -435,8 +605,8 @@ vector Record::items(const string & name) const return res; } - -vector Record::encodeInner() const +template +vector RecordT::encodeInner() const { vector res; auto inserter = std::back_inserter(res); @@ -471,6 +641,9 @@ vector Record::encodeInner() const return res; } +template class RecordT; +template class RecordT; + Blob::Blob(const vector & vec): ptr(make_shared>(vec)) @@ -486,15 +659,16 @@ vector Blob::encodeInner() const return *ptr; } -Blob Blob::decode(Storage, +Blob Blob::decode( vector::const_iterator begin, vector::const_iterator end) { return Blob(make_shared>(begin, end)); } -optional::const_iterator>> -Object::decodePrefix(Storage st, +template +optional, vector::const_iterator>> +ObjectT::decodePrefix(const S & st, vector::const_iterator begin, vector::const_iterator end) { @@ -512,11 +686,14 @@ Object::decodePrefix(Storage st, auto cend = newline + 1 + size; string type(begin, space); - optional obj; + optional> obj; if (type == "rec") - obj.emplace(Record::decode(st, newline + 1, cend)); + if (auto rec = RecordT::decode(st, newline + 1, cend)) + obj.emplace(*rec); + else + return nullopt; else if (type == "blob") - obj.emplace(Blob::decode(st, newline + 1, cend)); + obj.emplace(Blob::decode(newline + 1, cend)); else throw runtime_error("unknown object type '" + type + "'"); @@ -525,12 +702,14 @@ Object::decodePrefix(Storage st, return nullopt; } -optional Object::decode(Storage st, const vector & data) +template +optional> ObjectT::decode(const S & st, const vector & data) { return decode(st, data.begin(), data.end()); } -optional Object::decode(Storage st, +template +optional> ObjectT::decode(const S & st, vector::const_iterator begin, vector::const_iterator end) { @@ -542,7 +721,8 @@ optional Object::decode(Storage st, return nullopt; } -vector Object::encode() const +template +vector ObjectT::encode() const { vector res, inner; string type; @@ -569,25 +749,33 @@ vector Object::encode() const return res; } -optional Object::load(const Ref & ref) +template +optional> ObjectT::load(const typename S::Ref & ref) { - return *ref; + if (ref) + return *ref; + return nullopt; } -optional Object::asRecord() const +template +optional> ObjectT::asRecord() const { - if (holds_alternative(content)) - return std::get(content); + if (holds_alternative>(content)) + return std::get>(content); return nullopt; } -optional Object::asBlob() const +template +optional ObjectT::asBlob() const { if (holds_alternative(content)) return std::get(content); return nullopt; } +template class ObjectT; +template class ObjectT; + vector> erebos::collectStoredObjects(const Stored & from) { unordered_set seen; diff --git a/src/storage.h b/src/storage.h index 68002fa..cdb8984 100644 --- a/src/storage.h +++ b/src/storage.h @@ -3,35 +3,112 @@ #include "erebos/storage.h" #include +#include #include namespace fs = std::filesystem; using std::optional; using std::shared_future; +using std::shared_ptr; +using std::unique_ptr; +using std::unordered_map; using std::unordered_set; +using std::variant; using std::vector; namespace erebos { -struct Storage::Priv +class StorageBackend { - static constexpr size_t CHUNK = 16384; +public: + StorageBackend() = default; + virtual ~StorageBackend() = default; - fs::path root; + virtual bool contains(const Digest &) const = 0; + + virtual optional> loadBytes(const Digest &) const = 0; + virtual void storeBytes(const Digest &, const vector &) = 0; + + virtual optional> loadKey(const Digest &) const = 0; + virtual void storeKey(const Digest &, const vector &) = 0; +}; + +class FilesystemStorage : public StorageBackend +{ +public: + FilesystemStorage(const fs::path &); + virtual ~FilesystemStorage() = default; + + virtual bool contains(const Digest &) const override; + + virtual optional> loadBytes(const Digest &) const override; + virtual void storeBytes(const Digest &, const vector &) override; + + virtual optional> loadKey(const Digest &) const override; + virtual void storeKey(const Digest &, const vector &) override; + +private: + static constexpr size_t CHUNK = 16384; fs::path objectPath(const Digest &) const; fs::path keyPath(const Digest &) const; - optional> loadBytes(const Digest &) const; - void storeBytes(const Digest &, const vector &) const; + + fs::path root; }; -struct Ref::Priv +class MemoryStorage : public StorageBackend { - Storage storage; - Digest digest; +public: + MemoryStorage() = default; + virtual ~MemoryStorage() = default; + + virtual bool contains(const Digest &) const override; + + virtual optional> loadBytes(const Digest &) const override; + virtual void storeBytes(const Digest &, const vector &) override; - shared_future object; + virtual optional> loadKey(const Digest &) const override; + virtual void storeKey(const Digest &, const vector &) override; + +private: + unordered_map> storage; + unordered_map> keys; +}; + +class ChainStorage : public StorageBackend +{ +public: + ChainStorage(shared_ptr storage): + ChainStorage(std::move(storage), nullptr) {} + ChainStorage(shared_ptr storage, unique_ptr parent): + storage(std::move(storage)), parent(std::move(parent)) {} + virtual ~ChainStorage() = default; + + virtual bool contains(const Digest &) const override; + + virtual optional> loadBytes(const Digest &) const override; + virtual void storeBytes(const Digest &, const vector &) override; + + virtual optional> loadKey(const Digest &) const override; + virtual void storeKey(const Digest &, const vector &) override; + +private: + shared_ptr storage; + unique_ptr parent; +}; + +struct Storage::Priv +{ + shared_ptr backend; + + optional> loadBytes(const Digest & digest) const; +}; + +struct Ref::Priv +{ + const unique_ptr storage; + const Digest digest; }; vector> collectStoredObjects(const Stored &); -- cgit v1.2.3