fix(sentinel): lock push dedupe decisions
Guard Sentinel push hash checks and cache updates with a server-scoped atomic cache lock to prevent concurrent duplicate dispatches.
This commit is contained in:
parent
7677fac2f5
commit
b5be9fe9e8
2 changed files with 22 additions and 9 deletions
|
|
@ -117,19 +117,22 @@ private function shouldDispatchUpdate(Server $server, array $data): bool
|
||||||
$hash = $this->containerStateHash($data);
|
$hash = $this->containerStateHash($data);
|
||||||
$hashKey = "sentinel:push-hash:{$server->id}";
|
$hashKey = "sentinel:push-hash:{$server->id}";
|
||||||
$forceKey = "sentinel:push-force:{$server->id}";
|
$forceKey = "sentinel:push-force:{$server->id}";
|
||||||
|
$lockKey = "sentinel:push-lock:{$server->id}";
|
||||||
|
|
||||||
$cachedHash = Cache::get($hashKey);
|
return Cache::lock($lockKey, 10)->block(5, function () use ($hashKey, $forceKey, $hash): bool {
|
||||||
$forceActive = Cache::has($forceKey);
|
$cachedHash = Cache::get($hashKey);
|
||||||
|
$forceActive = Cache::has($forceKey);
|
||||||
|
|
||||||
$shouldDispatch = $cachedHash === null || $cachedHash !== $hash || ! $forceActive;
|
$shouldDispatch = $cachedHash === null || $cachedHash !== $hash || ! $forceActive;
|
||||||
|
|
||||||
if ($shouldDispatch) {
|
if ($shouldDispatch) {
|
||||||
// Day-long TTL bounds memory if a server stops pushing entirely.
|
// Day-long TTL bounds memory if a server stops pushing entirely.
|
||||||
Cache::put($hashKey, $hash, now()->addDay());
|
Cache::put($hashKey, $hash, now()->addDay());
|
||||||
Cache::put($forceKey, true, config('constants.sentinel.push_force_interval_seconds', 300));
|
Cache::put($forceKey, true, config('constants.sentinel.push_force_interval_seconds', 300));
|
||||||
}
|
}
|
||||||
|
|
||||||
return $shouldDispatch;
|
return $shouldDispatch;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,16 @@ function sentinelPayload(array $containers, ?float $diskPercentage = 42.0): arra
|
||||||
'empty containers' => [['containers' => []]],
|
'empty containers' => [['containers' => []]],
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
it('guards the dedupe decision with a server scoped atomic cache lock', function () {
|
||||||
|
$controller = file_get_contents(app_path('Http/Controllers/Api/SentinelController.php'));
|
||||||
|
|
||||||
|
expect($controller)
|
||||||
|
->toContain('$lockKey = "sentinel:push-lock:{$server->id}";')
|
||||||
|
->toContain('Cache::lock($lockKey, 10)->block(5, function () use ($hashKey, $forceKey, $hash): bool')
|
||||||
|
->toContain('Cache::put($hashKey, $hash, now()->addDay())')
|
||||||
|
->toContain("Cache::put(\$forceKey, true, config('constants.sentinel.push_force_interval_seconds', 300))");
|
||||||
|
});
|
||||||
|
|
||||||
it('dispatches the job when container state changes', function () use ($running) {
|
it('dispatches the job when container state changes', function () use ($running) {
|
||||||
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
|
pushSentinel($this->token, sentinelPayload($running()))->assertOk();
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue