365class RemoteServer :
public std::enable_shared_from_this<RemoteServer> {
387 ::morph::model::detail::ActionDispatcher& dispatcher MORPH_LIFETIMEBOUND =
388 ::morph::model::detail::defaultDispatcher(),
389 ::morph::model::detail::ModelRegistryFactory& registry MORPH_LIFETIMEBOUND =
390 ::morph::model::detail::defaultRegistry())
408 std::shared_ptr<::morph::session::IAuthorizer> authorizer,
409 ::morph::model::detail::ActionDispatcher& dispatcher MORPH_LIFETIMEBOUND =
410 ::morph::model::detail::defaultDispatcher(),
411 ::morph::model::detail::ModelRegistryFactory& registry MORPH_LIFETIMEBOUND =
412 ::morph::model::detail::defaultRegistry())
432 ::morph::model::detail::ActionDispatcher& dispatcher MORPH_LIFETIMEBOUND =
433 ::morph::model::detail::defaultDispatcher(),
434 ::morph::model::detail::ModelRegistryFactory& registry MORPH_LIFETIMEBOUND =
435 ::morph::model::detail::defaultRegistry())
436 :
RemoteServer{workerPool, ::morph::session::allowAllAuthorizer(), std::move(config), dispatcher, registry} {}
460 std::shared_ptr<::morph::session::IAuthorizer> authorizer,
ServerConfig config,
461 ::morph::model::detail::ActionDispatcher& dispatcher MORPH_LIFETIMEBOUND =
462 ::morph::model::detail::defaultDispatcher(),
463 ::morph::model::detail::ModelRegistryFactory& registry MORPH_LIFETIMEBOUND =
464 ::morph::model::detail::defaultRegistry())
465 : _strand{workerPool},
466 _strands{std::make_shared<::morph::exec::detail::ModelStrands>(workerPool)},
467 _dispatcher{dispatcher},
469 _authorizer{authorizer ? std::move(authorizer) : ::morph::session::allowAllAuthorizer()},
470 _config{validated(std::move(config))},
471 _executeTimeouts{_config.limits.executeTimeout.count() > 0
472 ? std::make_unique<::morph::async::detail::TimeoutScheduler>()
478 if (_config.healthHandler) {
479 _config.healthHandler(HealthStatus{.ready = true, .liveModels = 0, .inFlight = 0});
483 RemoteServer(
const RemoteServer&) =
delete;
484 RemoteServer& operator=(
const RemoteServer&) =
delete;
485 RemoteServer(RemoteServer&&) =
delete;
486 RemoteServer& operator=(RemoteServer&&) =
delete;
510 void handle(std::string msg, std::function<
void(std::string)> reply) {
511 handleImpl(std::move(msg), std::move(reply), 0);
529 handleImpl(std::move(msg), std::move(reply), cid);
564 Decoded decoded = decodeEnvelope(msg);
565 if (decoded.env && decoded.env->kind ==
"execute") {
566 return ::morph::wire::encode(::morph::wire::makeErr(
567 "handleInline does not support execute (reply is asynchronous)", decoded.env->callId));
570 std::function<void(std::string)> capture = [&out](std::string reply)
noexcept { out = std::move(reply); };
571 if (_strand.runningHere()) {
572 dispatchDecoded(std::move(decoded), capture, cid);
576 _strand.postTask([self = shared_from_this(), &decoded, &capture, cid, &done] {
580 self->dispatchDecoded(std::move(decoded), capture, cid);
605 ConnectionId const cid{_nextConnectionId.fetch_add(1) + 1};
606 _strand.postTask([self = shared_from_this(), cid] {
607 self->noteOwner(
"RemoteServer::openConnection");
608 self->_connectionScopes.try_emplace(cid);
641 _strand.postTask([self = shared_from_this(), cid] { self->closeConnectionHere(cid); });
661 _strand.postTask([self = shared_from_this(), cid, writer = std::move(writer)]()
mutable {
662 self->noteOwner(
"RemoteServer::setConnectionWriter");
663 if (!self->_connectionScopes.contains(cid)) {
667 self->discardChanges(cid);
668 self->_writers.erase(cid);
673 self->_writers[cid].writer = std::move(writer);
697 if (oversized(cid, envelope)) {
703 _strand.postTask([self = shared_from_this(), cid, envelope = std::move(envelope)]()
mutable {
704 self->postHere(cid, std::move(envelope));
725 _strand.postTask([self = shared_from_this(), cid] { self->connectionDrainedHere(cid); });
738 assert(_strand.runningHere() &&
"RemoteServer::eventsNegotiated runs on the server strand");
739 auto const found = _facts.find(cid);
740 return found != _facts.end() && found->second.events;
750 assert(_strand.runningHere() &&
"RemoteServer::watchTable runs on the server strand");
787 assert(_strand.runningHere() &&
"RemoteServer::fanOut runs on the server strand");
788 noteOwner(
"RemoteServer::fanOut");
789 std::map<ConnectionId, ::morph::wire::ChangesBody> bodies;
790 for (
auto const& change : changes.
records) {
791 for (
ConnectionId const cid : _watches.watchersOf(change.record)) {
793 bodies[cid].records.push_back(change);
798 addInstanceTopic(std::move(change), bodies);
800 Authentications authentications;
801 for (
auto& [cid, body] : bodies) {
802 bool const dropped = reauthorise(cid, body, caller, callerPrincipal, authentications);
803 if (!body.records.empty() || !body.instances.empty()) {
804 queueChanges(cid, body);
825 [[nodiscard]] PayloadCompleteness
payloadCompleteness() const noexcept {
return _config.payloadCompleteness; }
840 _strand.postTask([self = shared_from_this(), promise = std::move(settleable.second)]()
mutable {
841 self->noteOwner(
"RemoteServer::health");
842 promise.resolve(self->snapshotHealth());
844 return std::move(settleable.first);
862 _strand.postTask([self = shared_from_this()] {
863 self->noteOwner(
"RemoteServer::beginShutdown");
864 bool const wasReady = self->_ready;
865 self->_shuttingDown =
true;
866 self->_ready =
false;
867 if (wasReady && self->_config.healthHandler) {
868 self->_config.healthHandler(self->snapshotHealth());
896 [self = shared_from_this(), deadline,
898 self->noteOwner(
"RemoteServer::drainedWithin");
899 self->addDrainWaiter(deadline, promise);
901 return std::move(settleable.first);
906 using DrainPromise = std::shared_ptr<::morph::async::Completion<bool>::Promise>;
910 std::optional<::morph::wire::Envelope> env;
919 struct ConnectionOutbox {
920 ConnectionWriter writer;
921 bool owesResync{
false};
923 detail::ChangeCoalescer queued;
926 ::morph::async::detail::TimeoutScheduler::Handle windowTimer{0};
927 std::uint64_t window{0};
935 if (config.minProtocolVersion > config.maxProtocolVersion) {
936 throw std::invalid_argument(
"ServerConfig: minProtocolVersion must not exceed maxProtocolVersion");
938 if (config.coalesceWindow.count() < 0) {
939 throw std::invalid_argument(
"ServerConfig: coalesceWindow must not be negative");
941 if (std::ranges::find(config.services,
nullptr) != config.services.end()) {
942 throw std::invalid_argument(
"ServerConfig: services must not hold a null entry");
951 for (
const auto& service : _config.services) {
952 if (service->serves(kind)) {
953 return service.get();
962 static Decoded decodeEnvelope(std::string msg) {
965 decoded.env = ::morph::wire::decode(msg);
966 }
catch (
const std::exception& exc) {
967 decoded.raw = std::move(msg);
968 decoded.error = exc.what();
978 void handleImpl(std::string msg, std::function<
void(std::string)> reply,
ConnectionId cid) {
979 _strand.postTask([self = shared_from_this(), decoded = decodeEnvelope(std::move(msg)),
980 reply = std::move(reply),
981 cid]()
mutable {
self->dispatchDecoded(std::move(decoded), reply, cid); });
987 void noteOwner(
char const* site) {
988 ::morph::exec::detail::noteOwner(site, _strand.coreExecutor(), _strand.runningHere());
994 return HealthStatus{.ready = _ready, .liveModels = _instances.size(), .inFlight = _inFlight};
1001 noteOwner(
"RemoteServer::closeConnection");
1002 discardChanges(cid);
1003 _writers.erase(cid);
1005 _watches.dropConnection(cid);
1006 _principalBaselines.erase(cid);
1007 auto scopeIter = _connectionScopes.find(cid);
1008 if (scopeIter == _connectionScopes.end()) {
1011 for (
const auto& [mid, refs] : scopeIter->second) {
1015 for (std::size_t idx = 0; idx < refs; ++idx) {
1016 releaseInstance(mid);
1019 _connectionScopes.erase(scopeIter);
1031 if (cid == 0 || !_connectionScopes.contains(cid)) {
1033 return _throwawayFacts;
1043 static bool oversized(
ConnectionId cid, std::string
const& envelope) {
1044 if (envelope.size() <= ::morph::wire::kMaxEnvelopeBytes) {
1047 ::morph::observe::detail::emitMetric(::morph::observe::Metric::eventOversize, 1.0);
1048 ::morph::log::logWarn(
"[post] envelope for connection {} dropped: {} bytes exceeds {}", cid, envelope.size(),
1049 ::morph::wire::kMaxEnvelopeBytes);
1060 [[nodiscard]] std::vector<ConnectionId> scopesHolding(::morph::exec::detail::ModelId mid)
const {
1061 std::vector<ConnectionId> holders;
1062 for (
auto const& [cid, scope] : _connectionScopes) {
1063 if (scope.contains(mid)) {
1064 holders.push_back(cid);
1072 using Authentications = std::map<std::pair<std::string, std::string>,
bool>;
1085 [[nodiscard]]
bool stillAuthenticated(detail::Watch
const& watch, Authentications& asked)
const {
1086 auto [found, fresh] = asked.try_emplace({watch.principal, watch.context.token},
false);
1088 found->second = verifiedPrincipal(watch.context) == watch.principal;
1090 return found->second;
1100 void addInstanceTopic(::morph::effects::InstanceChange change,
1101 std::map<ConnectionId, ::morph::wire::ChangesBody>& bodies) {
1102 ::morph::exec::detail::ModelId
const mid{change.
instance};
1105 if (
auto const* inst = _instances.find(mid); inst !=
nullptr && inst->sharedKey) {
1106 change.
primary = inst->sharedKey->second;
1109 auto const recipients =
1110 change.
primary.empty() ? scopesHolding(mid) : _watches.watchersOf(change.modelType, change.primary);
1111 ::morph::wire::InstanceTopic
const topic{.modelType = std::move(change.
modelType),
1112 .primary = std::move(change.
primary),
1115 if (eventsNegotiated(cid)) {
1116 bodies[cid].instances.push_back(topic);
1152 std::string_view callerPrincipal, Authentications& asked) {
1153 bool dropped =
false;
1155 auto const authenticated = [&](detail::Watch
const& watch) {
1156 if (stillAuthenticated(watch, asked)) {
1159 std::string
const principal = watch.principal;
1160 std::string
const token = watch.context.token;
1161 _watches.dropSession(cid, principal, token);
1165 std::vector<::morph::effects::RecordChange> records;
1166 records.reserve(body.
records.size());
1167 for (
auto& change : body.
records) {
1168 auto const* watch = _watches.watchOf(cid, change.record);
1169 if (watch ==
nullptr || !authenticated(*watch)) {
1172 if (!mayWatchRecord(watch->context, change.record)) {
1173 _watches.remove(cid, ::morph::wire::WatchBody{.records = {change.record}, .instances = {}});
1177 bool const writer = cid == caller || (!callerPrincipal.empty() && watch->principal == callerPrincipal);
1178 hidePrivateKeys(change, writer);
1179 records.push_back(std::move(change));
1181 body.
records = std::move(records);
1182 std::vector<::morph::wire::InstanceTopic> instances;
1183 instances.reserve(body.
instances.size());
1187 if (watch ==
nullptr || !authenticated(*watch)) {
1190 if (!mayWatchInstance(watch->context, topic.
modelType)) {
1191 _watches.remove(cid, ::morph::wire::WatchBody{
1198 instances.push_back(std::move(topic));
1209 static void hidePrivateKeys(::morph::effects::RecordChange& change,
bool writer) {
1213 auto const* info = ::morph::effects::recordNamed(change.
record);
1214 if (info !=
nullptr && info->privateKeys) {
1215 change.
keys.reset();
1233 if (eventsNegotiated(cid)) {
1234 postHere(cid, ::morph::wire::encode(::morph::wire::makeEvent(::morph::wire::kResyncTopic,
"{}")));
1248 void queueChanges(
ConnectionId cid, ::morph::wire::ChangesBody
const& body) {
1249 auto found = _writers.find(cid);
1250 if (found == _writers.end() || found->second.owesResync) {
1253 ConnectionOutbox& outbox = found->second;
1254 outbox.queued.add(body);
1255 if (_config.coalesceWindow.count() == 0 || outbox.queued.keyCount() >= detail::kCoalesceKeyLimit) {
1259 if (outbox.window != 0) {
1262 if (!_strandTimer) {
1263 _strandTimer = std::make_unique<::morph::async::detail::TimeoutScheduler>();
1265 std::uint64_t
const window = ++_nextWindow;
1266 outbox.window = window;
1270 outbox.windowTimer = _strandTimer->schedule(_config.coalesceWindow, [weak = weak_from_this(), cid, window] {
1271 if (
auto self = weak.lock()) {
1272 self->_strand.postTask([self, cid, window] {
self->closeWindow(cid, window); });
1285 void closeWindow(
ConnectionId cid, std::uint64_t window) {
1286 noteOwner(
"RemoteServer::closeWindow");
1287 auto found = _writers.find(cid);
1288 if (found == _writers.end() || found->second.window != window) {
1291 found->second.window = 0;
1292 found->second.windowTimer = 0;
1301 auto found = _writers.find(cid);
1302 if (found == _writers.end()) {
1305 ConnectionOutbox& outbox = found->second;
1306 if (outbox.windowTimer != 0) {
1307 _strandTimer->cancel(outbox.windowTimer);
1310 outbox.windowTimer = 0;
1311 bool const queued = !outbox.queued.empty();
1312 (void)outbox.queued.take();
1320 auto found = _writers.find(cid);
1321 if (found == _writers.end()) {
1324 ConnectionOutbox& outbox = found->second;
1325 if (outbox.windowTimer != 0) {
1326 _strandTimer->cancel(outbox.windowTimer);
1329 outbox.windowTimer = 0;
1330 if (outbox.queued.empty()) {
1333 ::morph::wire::ChangesBody body = outbox.queued.take();
1334 std::string envelope = changesEnvelope(body);
1338 if (envelope.size() > ::morph::wire::kMaxEnvelopeBytes) {
1339 for (::morph::effects::RecordChange& change : body.
records) {
1340 change.
keys.reset();
1342 envelope = changesEnvelope(body);
1344 if (oversized(cid, envelope)) {
1345 envelope = ::morph::wire::encode(::morph::wire::makeEvent(::morph::wire::kResyncTopic,
"{}"));
1347 postHere(cid, std::move(envelope));
1353 [[nodiscard]]
static std::string changesEnvelope(::morph::wire::ChangesBody
const& body) {
1354 std::string payload;
1355 (void)glz::write_json(body, payload);
1356 return ::morph::wire::encode(::morph::wire::makeEvent(::morph::wire::kChangesTopic, std::move(payload)));
1362 void postHere(
ConnectionId cid, std::string envelope) {
1363 noteOwner(
"RemoteServer::post");
1364 auto found = _writers.find(cid);
1365 if (found == _writers.end()) {
1368 ConnectionOutbox& outbox = found->second;
1369 if (outbox.owesResync) {
1372 std::size_t backlog = 0;
1374 backlog = outbox.writer.backlog ? outbox.writer.backlog() : 0;
1375 }
catch (
const std::exception& exc) {
1376 writeFailed(cid, exc.what());
1379 writeFailed(cid,
"unknown exception");
1383 outbox.owesResync =
true;
1386 writeTo(cid, outbox, std::move(envelope));
1392 noteOwner(
"RemoteServer::connectionDrained");
1393 auto found = _writers.find(cid);
1394 if (found == _writers.end() || !found->second.owesResync) {
1397 found->second.owesResync =
false;
1398 writeTo(cid, found->second,
1399 ::morph::wire::encode(::morph::wire::makeEvent(::morph::wire::kResyncTopic,
"{}")));
1407 void writeTo(
ConnectionId cid, ConnectionOutbox& outbox, std::string envelope) {
1409 outbox.writer.write(std::move(envelope));
1410 }
catch (
const std::exception& exc) {
1411 writeFailed(cid, exc.what());
1413 writeFailed(cid,
"unknown exception");
1426 void writeFailed(
ConnectionId cid, std::string_view reason) {
1427 noteOwner(
"RemoteServer::writeFailed");
1428 ::morph::log::logWarn(
"[post] write to connection {} failed: {}", cid, reason);
1429 _watches.dropConnection(cid);
1444 void handleWatch(const ::morph::wire::Envelope& env,
const std::function<
void(std::string)>& reply,
1446 noteOwner(
"RemoteServer::watch");
1448 reply(::morph::wire::encode(::morph::wire::makeErr(
env.kind +
" needs a connection",
env.callId)));
1451 if (!_connectionScopes.contains(cid)) {
1452 reply(::morph::wire::encode(::morph::wire::makeErr(
"connection closed",
env.callId)));
1455 auto body = ::morph::wire::readWatchBody(
env.body);
1457 reply(::morph::wire::encode(::morph::wire::makeErr(body.error(),
env.callId)));
1460 ::morph::wire::WatchReply refused;
1461 if (
env.kind == ::morph::wire::kUnwatchKind) {
1462 _watches.remove(cid, *body);
1464 ::morph::wire::WatchBody admitted{.records = {}, .instances = {}, .replace = body->replace};
1465 std::set<std::string_view> seenRecords;
1466 for (
auto const& record : body->
records) {
1467 if (seenRecords.insert(record).second) {
1472 std::set<std::pair<std::string_view, std::string_view>> seenInstances;
1473 for (
auto const& topic : body->
instances) {
1480 _watches.add(cid, admitted, detail::Watch{.context =
env.session, .principal =
env.session.principal});
1481 std::ranges::move(overCap.refusedRecords, std::back_inserter(refused.
refusedRecords));
1482 std::ranges::move(overCap.refusedInstances, std::back_inserter(refused.
refusedInstances));
1485 (void)glz::write_json(refused, json);
1486 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, std::move(json))));
1502 template <
class Judge>
1503 [[nodiscard]]
bool cachedAdmission(const ::morph::session::Context& session, detail::AdmissionTopic kind,
1504 std::string_view name, Judge judge) {
1505 bool const cacheable = !
session.principal.empty();
1507 if (
auto const cached = _watches.admission(session, kind, name)) {
1511 bool const admitted = judge();
1513 _watches.noteAdmission(session, kind, name, admitted);
1532 [[nodiscard]]
bool mayWatchRecord(const ::morph::session::Context& session, std::string_view record) {
1533 if (::morph::effects::recordNamed(record) ==
nullptr) {
1536 return cachedAdmission(session, detail::AdmissionTopic::record, record, [&] {
1537 for (
auto const& actionId : ::morph::effects::actionsHolding(record)) {
1538 for (
auto const& modelType : _dispatcher.modelsOf(actionId)) {
1539 if (_authorizer->authorize(session, modelType, actionId)) {
1560 [[nodiscard]]
bool mayWatchInstance(const ::morph::session::Context& session, std::string_view modelType) {
1561 if (!_registry.contains(modelType)) {
1564 bool const mayRun = cachedAdmission(session, detail::AdmissionTopic::modelType, modelType, [&] {
1565 return std::ranges::any_of(_dispatcher.actionsOf(modelType), [&](std::string
const& actionId) {
1566 return _authorizer->authorize(session, modelType, actionId);
1569 return mayRun && _authorizer->authorizeInstance(session, modelType, {}, 0, {});
1575 void addDrainWaiter(std::chrono::milliseconds deadline,
const DrainPromise& promise) {
1576 if (_inFlight == 0) {
1577 promise->resolve(
true);
1580 if (deadline.count() <= 0) {
1581 promise->resolve(
false);
1584 std::uint64_t
const waiterId = ++_nextDrainWaiter;
1585 if (!_strandTimer) {
1586 _strandTimer = std::make_unique<::morph::async::detail::TimeoutScheduler>();
1591 auto const handle = _strandTimer->schedule(deadline, [weak = weak_from_this(), waiterId] {
1592 if (
auto self = weak.lock()) {
1593 self->_strand.postTask([self, waiterId] {
self->expireDrainWaiter(waiterId); });
1596 _drainWaiters.push_back(DrainWaiter{.id = waiterId, .promise = promise, .timer = handle});
1602 void expireDrainWaiter(std::uint64_t waiterId) {
1603 noteOwner(
"RemoteServer::drainedWithin");
1604 auto const found = std::ranges::find(_drainWaiters, waiterId, &DrainWaiter::id);
1605 if (found == _drainWaiters.end()) {
1608 DrainPromise
const promise = found->promise;
1609 _drainWaiters.erase(found);
1610 promise->resolve(
false);
1615 void executeFinished() {
1616 noteOwner(
"RemoteServer::executeFinished");
1618 ::morph::observe::detail::emitMetric(::morph::observe::Metric::executeInFlight,
1619 static_cast<double>(_inFlight));
1620 if (_inFlight != 0 || _drainWaiters.empty()) {
1623 auto waiters = std::move(_drainWaiters);
1624 _drainWaiters.clear();
1625 for (
auto& waiter : waiters) {
1626 _strandTimer->cancel(waiter.timer);
1627 waiter.promise->resolve(
true);
1636 std::uint64_t callId{0};
1640 std::shared_ptr<::core::async::StopSource> stop;
1647 void callFinished(
const CallTag& tag) {
1649 auto const found = _cancellable.find(CancelKey{.cid = tag.cid, .callId = tag.callId});
1650 if (found != _cancellable.end() && found->second.stop == tag.stop) {
1651 _cancellable.erase(found);
1669 void fileCancellable(
const CallTag& tag, const ::morph::wire::Envelope& env,
const std::string& owner) {
1670 _cancellable.insert_or_assign(CancelKey{.cid = tag.cid, .callId = tag.callId},
1671 CancelTarget{.stop = tag.stop,
1672 .principal =
env.session.principal,
1673 .modelType =
env.modelType,
1674 .actionType =
env.actionType,
1675 .modelId =
env.modelId,
1690 void cancelCall(const ::morph::wire::Envelope& env,
ConnectionId cid) {
1694 auto const found = _cancellable.find(CancelKey{.cid = cid, .callId =
env.cancelCallId});
1695 if (found == _cancellable.end()) {
1698 CancelTarget
const& target = found->second;
1699 if (
env.session.principal != target.principal ||
1700 !_authorizer->authorize(
env.session, target.modelType, target.actionType) ||
1701 !_authorizer->authorizeInstance(
env.session, target.modelType, target.actionType, target.modelId,
1705 auto const stop = target.stop;
1706 _cancellable.erase(found);
1710 static_cast<void>(stop->request_stop());
1733 void stampVerifiedPrincipal(::morph::wire::Envelope& env,
ConnectionId cid) {
1734 env.session.principal = verifiedPrincipal(
env.session);
1735 compareWithBaseline(
env.session.principal, cid);
1742 [[nodiscard]] std::string verifiedPrincipal(const ::morph::session::Context& session)
const {
1743 auto verified = _authorizer->authenticate(session);
1744 return verified ? std::move(*verified) : std::string{};
1762 void compareWithBaseline(
const std::string& principal,
ConnectionId cid) {
1763 if (cid == 0 || !_connectionScopes.contains(cid)) {
1766 auto const [baseline, first] = _principalBaselines.try_emplace(cid, principal);
1767 if (!first && baseline->second != principal) {
1768 bool const watched = _watches.topicCount(cid) != 0;
1769 bool const queued = discardChanges(cid);
1770 _watches.dropConnection(cid);
1771 baseline->second = principal;
1772 if (watched || queued) {
1786 void releaseInstance(::morph::exec::detail::ModelId mid) { (void)_instances.release(mid); }
1799 void releaseScoped(::morph::exec::detail::ModelId mid,
ConnectionId cid) {
1801 auto scopeIter = _connectionScopes.find(cid);
1802 if (scopeIter == _connectionScopes.end()) {
1805 auto refIter = scopeIter->second.find(mid);
1806 if (refIter == scopeIter->second.end()) {
1809 refIter->second -= 1;
1810 if (refIter->second == 0) {
1811 scopeIter->second.erase(refIter);
1814 releaseInstance(mid);
1821 bool noteScopeAttach(::morph::exec::detail::ModelId mid,
ConnectionId cid) {
1825 auto scopeIter = _connectionScopes.find(cid);
1826 if (scopeIter == _connectionScopes.end()) {
1829 scopeIter->second[mid] += 1;
1837 void attachLogIfConfigured(::morph::model::detail::IModelHolder& holder, const ::morph::wire::Envelope& env) {
1838 if (
env.contextKey.empty() || !_config.logProvider) {
1841 noteOwner(
"RemoteServer::attachLog");
1842 if (
auto log = _config.logProvider(
env.typeId,
env.contextKey)) {
1843 holder.attachActionLog(std::move(log),
env.contextKey);
1853 bool attachExisting(
const detail::DirectoryKey& dirKey, const ::morph::wire::Envelope& env,
1854 const std::function<
void(std::string)>& reply,
ConnectionId cid) {
1860 auto const attached = _instances.attach(dirKey);
1865 if (!noteScopeAttach(mid, cid)) {
1866 releaseInstance(mid);
1867 reply(::morph::wire::encode(::morph::wire::makeErr(
"connection closed",
env.callId)));
1870 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, {}, mid.v)));
1900 void acquireSharedInstance(const ::morph::wire::Envelope& env,
const std::function<
void(std::string)>& reply,
1901 ConnectionId cid, ::morph::exec::detail::ModelId releaseCurrent) {
1902 detail::DirectoryKey dirKey{
env.typeId,
env.primary};
1903 if (attachExisting(dirKey, env, reply, cid)) {
1908 if (releaseCurrent.v != 0U) {
1909 releaseScoped(releaseCurrent, cid);
1917 auto holder = _registry.create(
env.typeId,
env.primary);
1918 attachLogIfConfigured(*holder, env);
1922 if (attachExisting(dirKey, env, reply, cid)) {
1923 if (releaseCurrent.v != 0U) {
1924 releaseScoped(releaseCurrent, cid);
1928 ::morph::exec::detail::ModelId
const fresh{nextOpaqueId()};
1929 if (releaseCurrent.v != 0U) {
1930 releaseScoped(releaseCurrent, cid);
1933 if (limits.maxLiveModels != 0 && _instances.size() >= limits.maxLiveModels) {
1934 reply(::morph::wire::encode(::morph::wire::makeErr(
"too many models",
env.callId)));
1937 if (!noteScopeAttach(fresh, cid)) {
1938 reply(::morph::wire::encode(::morph::wire::makeErr(
"connection closed",
env.callId)));
1945 _instances.insertShared(fresh, std::move(holder), std::move(dirKey));
1946 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, {}, fresh.v)));
1963 void applyAssign(const ::morph::wire::Envelope& env) {
1964 if (
env.primary.empty()) {
1969 (void)_instances.promote(::morph::exec::detail::ModelId{env.modelId},
1970 detail::DirectoryKey{env.typeId, env.primary});
1994 [[nodiscard]] std::string missingRequiredFields(
const std::string& modelType,
const std::string& actionType,
1995 const std::string& body)
const {
1996 const auto* required = _dispatcher.requiredFieldsFor(modelType, actionType);
1997 if (required ==
nullptr || required->empty()) {
2000 glz::generic_u64 dom{};
2001 if (glz::read_json(dom, body) || !dom.is_object()) {
2004 std::string missing;
2005 for (
const auto& field : *required) {
2006 if (!dom.contains(field)) {
2007 if (!missing.empty()) {
2020 void handleInstances(const ::morph::wire::Envelope& env,
const std::function<
void(std::string)>& reply) {
2022 (void)glz::write_json(_instances.keysOfType(
env.typeId), body);
2023 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, std::move(body))));
2040 static void replyUndecodable(
const std::string& msg,
const std::string& error,
2041 const std::function<
void(std::string)>& reply,
ConnectionId cid) {
2042 constexpr std::size_t kLogPayloadPreviewBytes = 256;
2043 std::string_view
const preview =
2044 std::string_view{msg}.substr(0, std::min(msg.size(), kLogPayloadPreviewBytes));
2045 ::morph::log::logError(
2046 "[dispatchMessage] undecodable envelope from connection {}: {} ({} bytes, "
2047 "payload prefix: {}{})",
2048 cid, error, msg.size(), preview, msg.size() > kLogPayloadPreviewBytes ?
"..." :
"");
2049 reply(::morph::wire::encode(::morph::wire::makeErr(error)));
2058 void dispatchDecoded(Decoded decoded, std::function<
void(std::string)>& reply,
ConnectionId cid) {
2060 noteOwner(
"RemoteServer::dispatch");
2062 replyUndecodable(decoded.raw, decoded.error, reply, cid);
2066 dispatchEnvelope(std::move(*decoded.env), reply, cid);
2073 void dispatchEnvelope(::morph::wire::Envelope env, std::function<
void(std::string)>& reply,
ConnectionId cid) {
2082 ::morph::log::logDebug(
2083 "[dispatchMessage] connection {}: kind={} callId={} typeId={} modelId={} "
2084 "modelType={} actionType={} bodyBytes={}",
2085 cid,
env.kind,
env.callId,
env.typeId,
env.modelId,
env.modelType,
env.actionType,
env.body.size());
2090 if ((
env.kind ==
"register" ||
env.kind ==
"execute" ||
env.kind ==
"attach") && _shuttingDown) {
2091 reply(::morph::wire::encode(::morph::wire::makeErr(
"server shutting down",
env.callId)));
2095 if (
env.kind ==
"register") {
2098 ::morph::observe::detail::emitMetric(::morph::observe::Metric::registerCount, 1.0);
2099 if (
env.typeId.empty()) {
2100 throw std::runtime_error(
"register requires a typeId");
2110 if (limits.maxLiveModels != 0 && (!
env.shared ||
env.primary.empty()) &&
2111 _instances.size() >= limits.maxLiveModels) {
2112 reply(::morph::wire::encode(::morph::wire::makeErr(
"too many models",
env.callId)));
2122 stampVerifiedPrincipal(env, cid);
2127 if (!_authorizer->authorizeRegister(
env.session,
env.typeId)) {
2128 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2133 if (
env.shared && !
env.primary.empty()) {
2134 acquireSharedInstance(env, reply, cid, ::morph::exec::detail::ModelId{0});
2141 auto holder = _registry.create(
env.typeId,
env.primary);
2142 attachLogIfConfigured(*holder, env);
2150 if (cid != 0 && !_connectionScopes.contains(cid)) {
2151 reply(::morph::wire::encode(::morph::wire::makeErr(
"connection closed",
env.callId)));
2157 if (limits.maxLiveModels != 0 && _instances.size() >= limits.maxLiveModels) {
2158 reply(::morph::wire::encode(::morph::wire::makeErr(
"too many models",
env.callId)));
2161 ::morph::exec::detail::ModelId
const mid{nextOpaqueId()};
2163 _connectionScopes.at(cid)[mid] += 1;
2169 _instances.insertPrivate(mid, std::move(holder), std::move(
env.session.principal));
2170 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, {}, mid.v)));
2171 }
else if (
env.kind ==
"attach") {
2172 if (
env.typeId.empty()) {
2173 throw std::runtime_error(
"attach requires a typeId");
2175 stampVerifiedPrincipal(env, cid);
2176 if (!_authorizer->authorizeRegister(
env.session,
env.typeId)) {
2177 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2180 acquireSharedInstance(env, reply, cid, ::morph::exec::detail::ModelId{
env.modelId});
2181 }
else if (
env.kind ==
"assign") {
2182 if (
env.typeId.empty()) {
2183 throw std::runtime_error(
"assign requires a typeId");
2192 stampVerifiedPrincipal(env, cid);
2193 if (!_authorizer->authorizeRegister(
env.session,
env.typeId)) {
2194 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2198 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, {},
env.modelId)));
2199 }
else if (
env.kind ==
"instances") {
2200 if (
env.typeId.empty()) {
2201 throw std::runtime_error(
"instances requires a typeId");
2203 stampVerifiedPrincipal(env, cid);
2208 if (!_authorizer->authorize(
env.session,
env.typeId, {})) {
2209 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2212 handleInstances(env, reply);
2213 }
else if (
env.kind ==
"schemas") {
2214 if (
env.typeId.empty()) {
2215 throw std::runtime_error(
"schemas requires a typeId");
2217 stampVerifiedPrincipal(env, cid);
2225 if (!_authorizer->authorize(
env.session,
env.typeId, {})) {
2226 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2229 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, _dispatcher.schemasJson(
env.typeId))));
2230 }
else if (
env.kind ==
"deregister") {
2232 ::morph::observe::detail::emitMetric(::morph::observe::Metric::deregisterCount, 1.0);
2233 ::morph::exec::detail::ModelId
const mid{
env.modelId};
2238 stampVerifiedPrincipal(env, cid);
2242 if (!_authorizer->authorize(
env.session, {}, {})) {
2243 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2250 if (
const auto* inst = _instances.find(mid);
2251 inst !=
nullptr && !_authorizer->authorizeInstance(
env.session, {}, {}, mid.v, inst->owner)) {
2252 reply(::morph::wire::encode(::morph::wire::makeErr(
"unauthorized",
env.callId)));
2265 releaseScoped(mid, cid);
2266 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId)));
2267 }
else if (
env.kind ==
"execute") {
2268 dispatchExecute(std::move(env), reply, cid);
2269 }
else if (
env.kind ==
"cancel") {
2273 stampVerifiedPrincipal(env, cid);
2274 cancelCall(env, cid);
2278 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId)));
2279 }
else if (
env.kind ==
"hello") {
2281 const std::uint32_t minV = _config.minProtocolVersion;
2282 const std::uint32_t maxV = _config.maxProtocolVersion;
2283 if (
env.protocolVersion < minV ||
env.protocolVersion > maxV) {
2284 reply(::morph::wire::encode(::morph::wire::makeErr(
"protocol version unsupported",
env.callId)));
2287 (void)glz::write_json(
2288 ::morph::wire::ProtocolRange{
2289 .min = minV, .max = maxV, .capabilities = {std::string{::morph::wire::kCapabilityCancel}}},
2291 reply(::morph::wire::encode(::morph::wire::makeOk(
env.callId, std::move(body))));
2293 }
else if (
env.kind == ::morph::wire::kWatchKind ||
env.kind == ::morph::wire::kUnwatchKind) {
2296 stampVerifiedPrincipal(env, cid);
2297 handleWatch(env, reply, cid);
2302 stampVerifiedPrincipal(env, cid);
2303 reply(service->handle(env, *_authorizer, factsOf(cid)));
2305 reply(::morph::wire::encode(::morph::wire::makeErr(
"unknown envelope kind: " +
env.kind,
env.callId)));
2307 }
catch (
const std::exception& exc) {
2314 reply(::morph::wire::encode(::morph::wire::makeErr(exc.what(),
env.callId)));
2324 void dispatchExecute(::morph::wire::Envelope env, std::function<
void(std::string)> reply,
ConnectionId cid) {
2329 noteOwner(
"RemoteServer::admitExecute");
2330 auto reject = [&
env, &reply](
const char* message) {
2331 reply(::morph::wire::encode(::morph::wire::makeErr(message,
env.callId)));
2339 auto admission = _authorizer->admit(
env.session,
env.modelType,
env.actionType);
2340 std::string verified = admission.principal ? std::move(*admission.principal) : std::string{};
2341 compareWithBaseline(verified, cid);
2345 if (limits.maxInFlightExecutes != 0 && _inFlight >= limits.maxInFlightExecutes) {
2346 reject(
"server busy");
2349 if (!admission.allowed) {
2350 reject(
"unauthorized");
2360 env.session.principal = std::move(verified);
2361 ::morph::exec::detail::ModelId
const mid{
env.modelId};
2364 const auto* inst = _instances.find(mid);
2365 if (inst ==
nullptr) {
2369 reject(
"model not found");
2372 std::shared_ptr<::morph::model::detail::IModelHolder> holder = inst->holder;
2373 std::shared_ptr<detail::HydrationState> hydration = inst->hydration;
2379 if (!_authorizer->authorizeInstance(
env.session,
env.modelType,
env.actionType, mid.v, inst->owner)) {
2380 reject(
"unauthorized");
2388 if (_config.payloadCompleteness == PayloadCompleteness::RequireDeclaredFields) {
2389 const std::string missing = missingRequiredFields(
env.modelType,
env.actionType,
env.body);
2390 if (!missing.empty()) {
2391 const std::string message =
"payload missing required field(s): " + missing;
2392 reject(message.c_str());
2401 ::morph::observe::detail::emitMetric(::morph::observe::Metric::executeInFlight,
2402 static_cast<double>(_inFlight));
2411 bool const taskHandler = _dispatcher.dispatchesAsync(
env.modelType,
env.actionType);
2412 bool const cancellable = taskHandler && cid != 0;
2413 std::shared_ptr<::core::async::StopSource> stopSource;
2414 if (_executeTimeouts || cancellable) {
2415 stopSource = std::make_shared<::core::async::StopSource>();
2417 CallTag
const tag{.cid = cid, .callId =
env.callId, .stop = cancellable ? stopSource :
nullptr};
2418 auto self = shared_from_this();
2419 auto finished = std::make_shared<std::atomic_flag>();
2424 struct Reservation {
2425 RemoteServer& server;
2426 std::shared_ptr<std::atomic_flag> finished;
2428 bool handedOff =
false;
2429 Reservation(RemoteServer& owner, std::shared_ptr<std::atomic_flag> flag, CallTag call)
2430 : server{owner}, finished{std::move(flag)}, tag{std::move(call)} {}
2431 Reservation(
const Reservation&) =
delete;
2432 Reservation& operator=(
const Reservation&) =
delete;
2433 Reservation(Reservation&&) =
delete;
2434 Reservation& operator=(Reservation&&) =
delete;
2437 if (!handedOff && !finished->test_and_set()) {
2438 server.callFinished(tag);
2441 } reservation{*
this, finished, tag};
2443 fileCancellable(tag, env, inst->owner);
2446 std::uint64_t
const callId =
env.callId;
2447 auto replySlot = std::make_shared<std::function<void(std::string)>>(std::move(reply));
2450 auto const complete = [
self, finished, replySlot, tag](std::string msg) {
2451 if (!finished->test_and_set()) {
2452 self->_strand.postTask([self, tag] {
self->callFinished(tag); });
2453 (*replySlot)(std::move(msg));
2457 ::morph::async::detail::TimeoutScheduler::Handle timeoutHandle{};
2458 if (_executeTimeouts) {
2467 _executeTimeouts->schedule(limits.executeTimeout, [complete, callId, stopSource]()
mutable {
2468 complete(::morph::wire::encode(::morph::wire::makeErr(
"timeout", callId)));
2469 stopSource->request_stop();
2475 run.env = std::move(env);
2476 run.holder = std::move(holder);
2477 run.hydration = std::move(hydration);
2478 run.complete = complete;
2479 run.timeoutHandle = timeoutHandle;
2480 run.stopSource = std::move(stopSource);
2483 run.sharedKey = inst->sharedKey;
2494 auto shared = std::make_shared<RemoteRun>(std::move(run));
2496 [shared] { shared->holder->actionGate().enter([shared] { startTaskRemote(shared); }); });
2501 _strands->post(mid, [run = std::move(run)]()
mutable {
2502 auto& gate =
run.holder->actionGate();
2503 if (gate.tryEnter()) {
2507 gate.enter([waiting = std::make_shared<RemoteRun>(std::move(run))] { startRemote(*waiting); });
2512 reservation.handedOff =
true;
2526 [[nodiscard]] ::morph::exec::detail::ModelId nextOpaqueId() {
2528 if (uint64_t
const opaque = _idGen.permute(++_nextId); opaque != 0) {
2529 return ::morph::exec::detail::ModelId{opaque};
2541 std::shared_ptr<RemoteServer>
self;
2542 ::morph::wire::Envelope
env;
2543 std::shared_ptr<::morph::model::detail::IModelHolder> holder;
2544 std::shared_ptr<detail::HydrationState> hydration;
2545 std::function<void(std::string)> complete;
2546 ::morph::async::detail::TimeoutScheduler::Handle timeoutHandle{};
2548 std::shared_ptr<::core::async::StopSource> stopSource;
2549 ::morph::exec::detail::ModelId mid{};
2553 std::optional<detail::DirectoryKey> sharedKey;
2554 std::chrono::steady_clock::time_point start;
2555 ::morph::observe::SpanId spanId{};
2574 static void admitRemote(RemoteRun& run) {
2575 run.start = std::chrono::steady_clock::now();
2577 ::morph::observe::detail::beginSpan(
run.env.session.requestId,
run.env.modelType,
run.env.actionType);
2582 static void startRemote(RemoteRun& run) {
2587 std::exception_ptr error;
2589 ::morph::session::detail::ScopedContext
const scoped{
run.env.session};
2590 result =
run.self->_dispatcher.dispatch(
run.env.modelType,
run.env.actionType, *
run.holder,
run.env.body);
2592 error = std::current_exception();
2594 finishRemote(run, std::move(result), error);
2599 static void startTaskRemote(
const std::shared_ptr<RemoteRun>& run) {
2603 std::shared_ptr<::morph::exec::detail::TaskResumer> executor;
2605 ::morph::session::detail::ScopedContext
const scoped{
run->env.session};
2606 auto const& strands =
run->self->_strands;
2607 executor = std::make_shared<::morph::exec::detail::TaskResumer>(strands,
run->mid,
run->env.session);
2608 strands->enroll(
run->mid, executor);
2609 auto token =
run->stopSource ?
run->stopSource->get_token() : ::core::async::StopToken{};
2610 run->self->_dispatcher.dispatchAsync(
2611 run->env.modelType,
run->env.actionType, *
run->holder,
run->env.body, executor, std::move(token),
2612 [run, resumer = executor.get()](std::string result, std::exception_ptr error) {
2617 auto const held = run->self->_strands;
2618 held->withdraw(run->mid, resumer);
2619 held->runOnStrand(run->mid, [run, result = std::move(result), error = std::move(error)] {
2620 finishRemote(*run, result, error);
2624 run->self->_strands->withdraw(
run->mid, executor.get());
2625 finishRemote(*run, std::string{}, std::current_exception());
2637 static void postChanges(RemoteRun
const& run, std::string_view result) {
2640 run.env.actionType, result,
2642 .instance = run.mid.v,
2643 .primary = run.sharedKey ? run.sharedKey->second : std::string{}});
2644 if (changes.empty()) {
2648 run.self->_strand.postTask([self =
run.self, changes = std::move(changes), caller =
run.cid,
2649 principal =
run.env.session.principal]()
mutable {
2650 self->fanOut(std::move(changes), caller, principal);
2652 }
catch (
const std::exception& exc) {
2653 ::morph::log::logError(
"[fanOut] changes of {} not derived: {}",
run.env.actionType, exc.what());
2655 ::morph::log::logError(
"[fanOut] changes of {} not derived: unknown exception",
run.env.actionType);
2661 static void finishRemote(RemoteRun& run, std::string result,
const std::exception_ptr& error) {
2665 if (
self._executeTimeouts) {
2666 self._executeTimeouts->cancel(
run.timeoutHandle);
2668 bool const succeeded = error ==
nullptr;
2674 if (
run.hydration) {
2675 run.hydration->settle(succeeded);
2677 ::morph::observe::detail::endSpan(
run.spanId, succeeded);
2678 auto const elapsedMs =
2679 std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() -
run.start).count();
2680 std::array<std::pair<std::string_view, std::string_view>, 2>
const tags{
2681 {{
"modelType",
run.env.modelType}, {
"actionType",
run.env.actionType}}};
2682 ::morph::observe::detail::emitMetric(::morph::observe::Metric::executeLatencyMs, elapsedMs, tags);
2684 postChanges(run, result);
2685 run.complete(::morph::wire::encode(::morph::wire::makeOk(
run.env.callId, std::move(result))));
2687 ::morph::observe::detail::emitMetric(::morph::observe::Metric::executeErrors, 1.0, tags);
2688 std::string message =
"unknown exception";
2690 std::rethrow_exception(error);
2691 }
catch (
const std::exception& exc) {
2692 message = exc.what();
2697 run.complete(::morph::wire::encode(::morph::wire::makeErr(message,
run.env.callId)));
2699 run.holder->actionGate().leave();
2708 std::shared_ptr<::morph::exec::detail::ModelStrands> _strands;
2709 ::morph::model::detail::ActionDispatcher& _dispatcher;
2710 ::morph::model::detail::ModelRegistryFactory& _registry;
2711 std::shared_ptr<::morph::session::IAuthorizer> _authorizer;
2714 ServerConfig
const _config;
2718 std::unique_ptr<::morph::async::detail::TimeoutScheduler>
const _executeTimeouts;
2728 detail::InstanceDirectory _instances;
2735 std::unordered_map<
ConnectionId, std::unordered_map<::morph::exec::detail::ModelId, std::size_t,
2736 ::morph::exec::detail::ModelIdHash>>
2740 std::unordered_map<ConnectionId, ConnectionOutbox> _writers;
2743 std::unordered_map<ConnectionId, ConnectionFacts> _facts;
2746 ConnectionFacts _throwawayFacts;
2749 detail::WatchTable _watches;
2754 std::unordered_map<ConnectionId, std::string> _principalBaselines;
2759 std::uint64_t callId{0};
2760 bool operator==(
const CancelKey&)
const =
default;
2762 struct CancelKeyHash {
2763 std::size_t operator()(
const CancelKey& key)
const noexcept {
2766 constexpr std::uint64_t kSpread = 0x9E3779B97F4A7C15ULL;
2767 return std::hash<std::uint64_t>{}((key.cid * kSpread) ^ key.callId);
2771 struct CancelTarget {
2772 std::shared_ptr<::core::async::StopSource> stop;
2773 std::string principal;
2774 std::string modelType;
2775 std::string actionType;
2776 std::uint64_t modelId{0};
2782 std::unordered_map<CancelKey, CancelTarget, CancelKeyHash> _cancellable;
2783 std::uint64_t _nextId{0};
2784 detail::OpaqueIdGenerator _idGen;
2789 std::size_t _inFlight{0};
2791 bool _shuttingDown{
false};
2795 struct DrainWaiter {
2797 DrainPromise promise;
2798 ::morph::async::detail::TimeoutScheduler::Handle timer;
2800 std::vector<DrainWaiter> _drainWaiters;
2801 std::uint64_t _nextDrainWaiter{0};
2804 std::unique_ptr<::morph::async::detail::TimeoutScheduler> _strandTimer;
2807 std::uint64_t _nextWindow{0};
2810 std::atomic<uint64_t> _nextConnectionId{0};