Revert "feat(notifications): deduplicate repeated email alerts"
This reverts commit 8c1405e168.
This commit is contained in:
parent
8c1405e168
commit
2a0183bfad
33 changed files with 14 additions and 516 deletions
|
|
@ -29,16 +29,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('api_token_expiring');
|
return $notifiable->getEnabledChannels('api_token_expiring');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "api-token-expiring:{$this->token->id}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 172800;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -52,16 +52,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('deployment_failure');
|
return $notifiable->getEnabledChannels('deployment_failure');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "deployment-failed:{$this->deployment_uuid}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -52,16 +52,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('deployment_success');
|
return $notifiable->getEnabledChannels('deployment_success');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "deployment-success:{$this->deployment_uuid}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -49,16 +49,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('status_change');
|
return $notifiable->getEnabledChannels('status_change');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "restart-limit-reached:application:{$this->resource->uuid}:count:{$this->restart_count}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -42,16 +42,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('status_change');
|
return $notifiable->getEnabledChannels('status_change');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "application-status-changed:application:{$this->resource->uuid}:stopped";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
@ -60,7 +50,6 @@ public function toMail(): MailMessage
|
||||||
$mail->view('emails.application-status-changes', [
|
$mail->view('emails.application-status-changes', [
|
||||||
'name' => $this->resource_name,
|
'name' => $this->resource_name,
|
||||||
'fqdn' => $fqdn,
|
'fqdn' => $fqdn,
|
||||||
'application_url' => $this->resource_url,
|
|
||||||
'resource_url' => $this->resource_url,
|
'resource_url' => $this->resource_url,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,20 +4,13 @@
|
||||||
|
|
||||||
use App\Exceptions\NonReportableException;
|
use App\Exceptions\NonReportableException;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
use App\Services\NotificationDeduplicator;
|
|
||||||
use Exception;
|
use Exception;
|
||||||
use Illuminate\Notifications\Notification;
|
use Illuminate\Notifications\Notification;
|
||||||
use Resend;
|
use Resend;
|
||||||
use Resend\Exceptions\ErrorException;
|
|
||||||
use Resend\Exceptions\TransporterException;
|
|
||||||
use Symfony\Component\Mailer\Mailer;
|
|
||||||
use Symfony\Component\Mailer\Transport\Smtp\EsmtpTransport;
|
|
||||||
use Symfony\Component\Mime\Address;
|
|
||||||
use Symfony\Component\Mime\Email;
|
|
||||||
|
|
||||||
class EmailChannel
|
class EmailChannel
|
||||||
{
|
{
|
||||||
public function __construct(private NotificationDeduplicator $deduplicator) {}
|
public function __construct() {}
|
||||||
|
|
||||||
public function send(SendsEmail $notifiable, Notification $notification): void
|
public function send(SendsEmail $notifiable, Notification $notification): void
|
||||||
{
|
{
|
||||||
|
|
@ -74,11 +67,6 @@ public function send(SendsEmail $notifiable, Notification $notification): void
|
||||||
}
|
}
|
||||||
|
|
||||||
$mailMessage = $notification->toMail($notifiable);
|
$mailMessage = $notification->toMail($notifiable);
|
||||||
$renderedMail = (string) $mailMessage->render();
|
|
||||||
|
|
||||||
if (! $this->deduplicator->shouldSend($notifiable, $notification, self::class, $recipients, $mailMessage->subject, $renderedMail)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($isResendEnabled) {
|
if ($isResendEnabled) {
|
||||||
$resend = Resend::client($settings->resend_api_key);
|
$resend = Resend::client($settings->resend_api_key);
|
||||||
|
|
@ -87,17 +75,17 @@ public function send(SendsEmail $notifiable, Notification $notification): void
|
||||||
'from' => $from,
|
'from' => $from,
|
||||||
'to' => $recipients,
|
'to' => $recipients,
|
||||||
'subject' => $mailMessage->subject,
|
'subject' => $mailMessage->subject,
|
||||||
'html' => $renderedMail,
|
'html' => (string) $mailMessage->render(),
|
||||||
]);
|
]);
|
||||||
} elseif ($isSmtpEnabled) {
|
} elseif ($isSmtpEnabled) {
|
||||||
$encryption = match (strtolower($settings->smtp_encryption ?? '')) {
|
$encryption = match (strtolower($settings->smtp_encryption)) {
|
||||||
'starttls' => null,
|
'starttls' => null,
|
||||||
'tls' => 'tls',
|
'tls' => 'tls',
|
||||||
'none' => null,
|
'none' => null,
|
||||||
default => null,
|
default => null,
|
||||||
};
|
};
|
||||||
|
|
||||||
$transport = new EsmtpTransport(
|
$transport = new \Symfony\Component\Mailer\Transport\Smtp\EsmtpTransport(
|
||||||
$settings->smtp_host,
|
$settings->smtp_host,
|
||||||
$settings->smtp_port,
|
$settings->smtp_port,
|
||||||
$encryption
|
$encryption
|
||||||
|
|
@ -105,20 +93,20 @@ public function send(SendsEmail $notifiable, Notification $notification): void
|
||||||
$transport->setUsername($settings->smtp_username ?? '');
|
$transport->setUsername($settings->smtp_username ?? '');
|
||||||
$transport->setPassword($settings->smtp_password ?? '');
|
$transport->setPassword($settings->smtp_password ?? '');
|
||||||
|
|
||||||
$mailer = new Mailer($transport);
|
$mailer = new \Symfony\Component\Mailer\Mailer($transport);
|
||||||
|
|
||||||
$fromEmail = $settings->smtp_from_address ?? 'noreply@localhost';
|
$fromEmail = $settings->smtp_from_address ?? 'noreply@localhost';
|
||||||
$fromName = $settings->smtp_from_name ?? 'System';
|
$fromName = $settings->smtp_from_name ?? 'System';
|
||||||
$from = new Address($fromEmail, $fromName);
|
$from = new \Symfony\Component\Mime\Address($fromEmail, $fromName);
|
||||||
$email = (new Email)
|
$email = (new \Symfony\Component\Mime\Email)
|
||||||
->from($from)
|
->from($from)
|
||||||
->to(...$recipients)
|
->to(...$recipients)
|
||||||
->subject($mailMessage->subject)
|
->subject($mailMessage->subject)
|
||||||
->html($renderedMail);
|
->html((string) $mailMessage->render());
|
||||||
|
|
||||||
$mailer->send($email);
|
$mailer->send($email);
|
||||||
}
|
}
|
||||||
} catch (ErrorException $e) {
|
} catch (\Resend\Exceptions\ErrorException $e) {
|
||||||
// Map HTTP status codes to user-friendly messages
|
// Map HTTP status codes to user-friendly messages
|
||||||
$userMessage = match ($e->getErrorCode()) {
|
$userMessage = match ($e->getErrorCode()) {
|
||||||
403 => 'Invalid Resend API key. Please verify your API key in the Resend dashboard and update it in settings.',
|
403 => 'Invalid Resend API key. Please verify your API key in the Resend dashboard and update it in settings.',
|
||||||
|
|
@ -143,13 +131,13 @@ public function send(SendsEmail $notifiable, Notification $notification): void
|
||||||
|
|
||||||
// Don't report expected errors (invalid keys, validation) to Sentry
|
// Don't report expected errors (invalid keys, validation) to Sentry
|
||||||
if (in_array($e->getErrorCode(), [403, 401, 400])) {
|
if (in_array($e->getErrorCode(), [403, 401, 400])) {
|
||||||
throw NonReportableException::fromException(new Exception($userMessage, $e->getCode(), $e));
|
throw NonReportableException::fromException(new \Exception($userMessage, $e->getCode(), $e));
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new Exception($userMessage, $e->getCode(), $e);
|
throw new \Exception($userMessage, $e->getCode(), $e);
|
||||||
} catch (TransporterException $e) {
|
} catch (\Resend\Exceptions\TransporterException $e) {
|
||||||
send_internal_notification("Resend Transport Error: {$e->getMessage()}");
|
send_internal_notification("Resend Transport Error: {$e->getMessage()}");
|
||||||
throw new Exception('Unable to connect to Resend API. Please check your internet connection and try again.');
|
throw new \Exception('Unable to connect to Resend API. Please check your internet connection and try again.');
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
// Check if this is a Resend domain verification error on cloud instances
|
// Check if this is a Resend domain verification error on cloud instances
|
||||||
if (isCloud() && str_contains($e->getMessage(), 'domain is not verified')) {
|
if (isCloud() && str_contains($e->getMessage(), 'domain is not verified')) {
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@
|
||||||
namespace App\Notifications\Channels;
|
namespace App\Notifications\Channels;
|
||||||
|
|
||||||
use App\Models\User;
|
use App\Models\User;
|
||||||
use App\Services\NotificationDeduplicator;
|
|
||||||
use Exception;
|
use Exception;
|
||||||
use Illuminate\Mail\Message;
|
use Illuminate\Mail\Message;
|
||||||
use Illuminate\Notifications\Notification;
|
use Illuminate\Notifications\Notification;
|
||||||
|
|
@ -11,8 +10,6 @@
|
||||||
|
|
||||||
class TransactionalEmailChannel
|
class TransactionalEmailChannel
|
||||||
{
|
{
|
||||||
public function __construct(private NotificationDeduplicator $deduplicator) {}
|
|
||||||
|
|
||||||
public function send(User $notifiable, Notification $notification): void
|
public function send(User $notifiable, Notification $notification): void
|
||||||
{
|
{
|
||||||
$settings = instanceSettings();
|
$settings = instanceSettings();
|
||||||
|
|
@ -30,19 +27,13 @@ public function send(User $notifiable, Notification $notification): void
|
||||||
}
|
}
|
||||||
$this->bootConfigs();
|
$this->bootConfigs();
|
||||||
$mailMessage = $notification->toMail($notifiable);
|
$mailMessage = $notification->toMail($notifiable);
|
||||||
$renderedMail = (string) $mailMessage->render();
|
|
||||||
|
|
||||||
if (! $this->deduplicator->shouldSend($notifiable, $notification, self::class, [$email], $mailMessage->subject, $renderedMail)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
Mail::send(
|
Mail::send(
|
||||||
[],
|
[],
|
||||||
[],
|
[],
|
||||||
fn (Message $message) => $message
|
fn (Message $message) => $message
|
||||||
->to($email)
|
->to($email)
|
||||||
->subject($mailMessage->subject)
|
->subject($mailMessage->subject)
|
||||||
->html($renderedMail)
|
->html((string) $mailMessage->render())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('status_change');
|
return $notifiable->getEnabledChannels('status_change');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "container-restarted:server:{$this->server->uuid}:container:{$this->name}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('status_change');
|
return $notifiable->getEnabledChannels('status_change');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "container-stopped:server:{$this->server->uuid}:container:{$this->name}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -15,19 +15,4 @@ class CustomEmailNotification extends Notification implements ShouldQueue
|
||||||
public $tries = 5;
|
public $tries = 5;
|
||||||
|
|
||||||
public $maxExceptions = 5;
|
public $maxExceptions = 5;
|
||||||
|
|
||||||
public function shouldDeduplicate(): bool
|
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 900;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,6 @@
|
||||||
|
|
||||||
class BackupFailed extends CustomEmailNotification
|
class BackupFailed extends CustomEmailNotification
|
||||||
{
|
{
|
||||||
public int|string|null $backupId = null;
|
|
||||||
|
|
||||||
public string $name;
|
public string $name;
|
||||||
|
|
||||||
public string $frequency;
|
public string $frequency;
|
||||||
|
|
@ -20,7 +18,6 @@ class BackupFailed extends CustomEmailNotification
|
||||||
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $output, public $database_name)
|
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $output, public $database_name)
|
||||||
{
|
{
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
$this->backupId = data_get($backup, 'uuid') ?? data_get($backup, 'id');
|
|
||||||
$this->name = $database->name;
|
$this->name = $database->name;
|
||||||
$this->frequency = $backup->frequency;
|
$this->frequency = $backup->frequency;
|
||||||
}
|
}
|
||||||
|
|
@ -30,16 +27,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('backup_failure');
|
return $notifiable->getEnabledChannels('backup_failure');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "backup-failed:backup:{$this->backupId}:database:{$this->database->uuid}:output:".hash('sha256', (string) $this->output);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 21600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,6 @@
|
||||||
|
|
||||||
class BackupSuccess extends CustomEmailNotification
|
class BackupSuccess extends CustomEmailNotification
|
||||||
{
|
{
|
||||||
public int|string|null $backupId = null;
|
|
||||||
|
|
||||||
public string $name;
|
public string $name;
|
||||||
|
|
||||||
public string $frequency;
|
public string $frequency;
|
||||||
|
|
@ -20,7 +18,6 @@ class BackupSuccess extends CustomEmailNotification
|
||||||
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name)
|
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name)
|
||||||
{
|
{
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
$this->backupId = data_get($backup, 'uuid') ?? data_get($backup, 'id');
|
|
||||||
|
|
||||||
$this->name = $database->name;
|
$this->name = $database->name;
|
||||||
$this->frequency = $backup->frequency;
|
$this->frequency = $backup->frequency;
|
||||||
|
|
@ -31,16 +28,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('backup_success');
|
return $notifiable->getEnabledChannels('backup_success');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "backup-success:backup:{$this->backupId}:database:{$this->database->uuid}:name:{$this->database_name}:frequency:{$this->frequency}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,6 @@
|
||||||
|
|
||||||
class BackupSuccessWithS3Warning extends CustomEmailNotification
|
class BackupSuccessWithS3Warning extends CustomEmailNotification
|
||||||
{
|
{
|
||||||
public int|string|null $backupId = null;
|
|
||||||
|
|
||||||
public string $name;
|
public string $name;
|
||||||
|
|
||||||
public string $frequency;
|
public string $frequency;
|
||||||
|
|
@ -22,7 +20,6 @@ class BackupSuccessWithS3Warning extends CustomEmailNotification
|
||||||
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error)
|
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error)
|
||||||
{
|
{
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
$this->backupId = data_get($backup, 'uuid') ?? data_get($backup, 'id');
|
|
||||||
|
|
||||||
$this->name = $database->name;
|
$this->name = $database->name;
|
||||||
$this->frequency = $backup->frequency;
|
$this->frequency = $backup->frequency;
|
||||||
|
|
@ -37,16 +34,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('backup_failure');
|
return $notifiable->getEnabledChannels('backup_failure');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "backup-s3-warning:backup:{$this->backupId}:database:{$this->database->uuid}:error:".hash('sha256', (string) $this->s3_error);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 21600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -28,16 +28,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('scheduled_task_failure');
|
return $notifiable->getEnabledChannels('scheduled_task_failure');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "scheduled-task-failed:task:{$this->task->uuid}:output:".hash('sha256', $this->output);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -28,16 +28,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('scheduled_task_success');
|
return $notifiable->getEnabledChannels('scheduled_task_success');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "scheduled-task-success:task:{$this->task->uuid}:output:".hash('sha256', $this->output);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('docker_cleanup_failure');
|
return $notifiable->getEnabledChannels('docker_cleanup_failure');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "docker-cleanup-failed:server:{$this->server->uuid}:message:".hash('sha256', $this->message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 21600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('docker_cleanup_success');
|
return $notifiable->getEnabledChannels('docker_cleanup_success');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "docker-cleanup-success:server:{$this->server->uuid}:message:".hash('sha256', $this->message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 21600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('server_force_disabled');
|
return $notifiable->getEnabledChannels('server_force_disabled');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "server-force-disabled:{$this->server->uuid}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('server_force_enabled');
|
return $notifiable->getEnabledChannels('server_force_enabled');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "server-force-enabled:{$this->server->uuid}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('hetzner_deletion_failed');
|
return $notifiable->getEnabledChannels('hetzner_deletion_failed');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "hetzner-deletion-failed:{$this->hetznerServerId}:error:".hash('sha256', $this->errorMessage);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('server_disk_usage');
|
return $notifiable->getEnabledChannels('server_disk_usage');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "high-disk-usage:server:{$this->server->uuid}:threshold:{$this->server_disk_usage_notification_threshold}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 21600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -30,16 +30,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('server_reachable');
|
return $notifiable->getEnabledChannels('server_reachable');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "server-reachable:{$this->server->uuid}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 1800;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -24,16 +24,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('server_patch');
|
return $notifiable->getEnabledChannels('server_patch');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "server-patch-check:server:{$this->server->uuid}:state:".hash('sha256', json_encode($this->patchData));
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail($notifiable = null): MailMessage
|
public function toMail($notifiable = null): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -38,18 +38,6 @@ private function getUpgradeTarget(array $info): string
|
||||||
return $this->formatVersion($info['latest'] ?? 'unknown');
|
return $this->formatVersion($info['latest'] ?? 'unknown');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
$serverUuids = $this->servers->pluck('uuid')->sort()->values()->join('|');
|
|
||||||
|
|
||||||
return 'traefik-version-outdated:servers:'.hash('sha256', $serverUuids);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail($notifiable = null): MailMessage
|
public function toMail($notifiable = null): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -30,16 +30,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('server_unreachable');
|
return $notifiable->getEnabledChannels('server_unreachable');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "server-unreachable:{$this->server->uuid}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): ?MailMessage
|
public function toMail(): ?MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -59,22 +59,6 @@ public function via(object $notifiable): array
|
||||||
return $notifiable->getEnabledChannels('ssl_certificate_renewal');
|
return $notifiable->getEnabledChannels('ssl_certificate_renewal');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
$resourceKeys = $this->resources
|
|
||||||
->map(fn ($resource) => data_get($resource, 'uuid') ?? data_get($resource, 'name'))
|
|
||||||
->sort()
|
|
||||||
->values()
|
|
||||||
->join('|');
|
|
||||||
|
|
||||||
return 'ssl-certificate-renewed:resources:'.hash('sha256', $resourceKeys);
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 86400;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$mail = new MailMessage;
|
$mail = new MailMessage;
|
||||||
|
|
|
||||||
|
|
@ -30,11 +30,6 @@ public function __construct(public ?string $emails = null, public ?string $chann
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function shouldDeduplicate(): bool
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function via(object $notifiable): array
|
public function via(object $notifiable): array
|
||||||
{
|
{
|
||||||
if ($this->channel) {
|
if ($this->channel) {
|
||||||
|
|
|
||||||
|
|
@ -25,16 +25,6 @@ public function __construct(
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "email-change-verification:user:{$this->user->id}:email:{$this->newEmail}:code:{$this->verificationCode}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return (int) max(1, now()->diffInSeconds($this->expiresAt, false));
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
// Use the configured expiry minutes value
|
// Use the configured expiry minutes value
|
||||||
|
|
|
||||||
|
|
@ -21,16 +21,6 @@ public function __construct(public User $user, public bool $isTransactionalEmail
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return "invitation-link:user:{$this->user->id}:email:{$this->user->email}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return 3600;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
public function toMail(): MailMessage
|
||||||
{
|
{
|
||||||
$invitation = TeamInvitation::whereEmail($this->user->email)->first();
|
$invitation = TeamInvitation::whereEmail($this->user->email)->first();
|
||||||
|
|
|
||||||
|
|
@ -15,11 +15,6 @@ public function __construct(public string $emails, public bool $isTransactionalE
|
||||||
$this->onQueue('high');
|
$this->onQueue('high');
|
||||||
}
|
}
|
||||||
|
|
||||||
public function shouldDeduplicate(): bool
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function via(): array
|
public function via(): array
|
||||||
{
|
{
|
||||||
return [EmailChannel::class];
|
return [EmailChannel::class];
|
||||||
|
|
|
||||||
|
|
@ -1,95 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Services;
|
|
||||||
|
|
||||||
use Illuminate\Notifications\Notification;
|
|
||||||
use Illuminate\Support\Facades\Cache;
|
|
||||||
|
|
||||||
class NotificationDeduplicator
|
|
||||||
{
|
|
||||||
public const DEFAULT_TTL = 900;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<int, string> $recipients
|
|
||||||
*/
|
|
||||||
public function shouldSend(object $notifiable, Notification $notification, string $channel, array $recipients, ?string $subject = null, ?string $body = null): bool
|
|
||||||
{
|
|
||||||
if (method_exists($notification, 'shouldDeduplicate') && ! $notification->shouldDeduplicate()) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
$ttl = method_exists($notification, 'deduplicateFor')
|
|
||||||
? $notification->deduplicateFor()
|
|
||||||
: self::DEFAULT_TTL;
|
|
||||||
|
|
||||||
if ($ttl <= 0) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return Cache::add(
|
|
||||||
$this->cacheKey($notifiable, $notification, $channel, $recipients, $subject, $body),
|
|
||||||
true,
|
|
||||||
$ttl,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<int, string> $recipients
|
|
||||||
*/
|
|
||||||
private function cacheKey(object $notifiable, Notification $notification, string $channel, array $recipients, ?string $subject, ?string $body): string
|
|
||||||
{
|
|
||||||
$semanticKey = method_exists($notification, 'deduplicationKey')
|
|
||||||
? $notification->deduplicationKey($notifiable, $channel)
|
|
||||||
: null;
|
|
||||||
|
|
||||||
$payload = $semanticKey
|
|
||||||
? $this->semanticFingerprint($notifiable, $notification, $channel, $recipients, $semanticKey)
|
|
||||||
: $this->defaultFingerprint($notifiable, $notification, $channel, $recipients, $subject, $body);
|
|
||||||
|
|
||||||
return 'notification-dedupe:'.hash('sha256', $payload);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<int, string> $recipients
|
|
||||||
*/
|
|
||||||
private function semanticFingerprint(object $notifiable, Notification $notification, string $channel, array $recipients, string $semanticKey): string
|
|
||||||
{
|
|
||||||
return json_encode([
|
|
||||||
'notification' => $notification::class,
|
|
||||||
'notifiable' => $notifiable::class,
|
|
||||||
'notifiable_id' => data_get($notifiable, 'id'),
|
|
||||||
'channel' => $channel,
|
|
||||||
'recipients' => $this->normalizeRecipients($recipients),
|
|
||||||
'semantic_key' => $semanticKey,
|
|
||||||
], JSON_THROW_ON_ERROR);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<int, string> $recipients
|
|
||||||
*/
|
|
||||||
private function defaultFingerprint(object $notifiable, Notification $notification, string $channel, array $recipients, ?string $subject, ?string $body): string
|
|
||||||
{
|
|
||||||
return json_encode([
|
|
||||||
'notification' => $notification::class,
|
|
||||||
'notifiable' => $notifiable::class,
|
|
||||||
'notifiable_id' => data_get($notifiable, 'id'),
|
|
||||||
'channel' => $channel,
|
|
||||||
'recipients' => $this->normalizeRecipients($recipients),
|
|
||||||
'subject' => $subject,
|
|
||||||
'body_hash' => hash('sha256', (string) $body),
|
|
||||||
], JSON_THROW_ON_ERROR);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param array<int, string> $recipients
|
|
||||||
* @return array<int, string>
|
|
||||||
*/
|
|
||||||
private function normalizeRecipients(array $recipients): array
|
|
||||||
{
|
|
||||||
return collect($recipients)
|
|
||||||
->map(fn (string $recipient) => mb_strtolower(trim($recipient)))
|
|
||||||
->sort()
|
|
||||||
->values()
|
|
||||||
->all();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -48,24 +48,6 @@ function applicationWithRestartState(array $attributes = []): Application
|
||||||
expect($html)->not->toContain('Stopped after reaching restart limit');
|
expect($html)->not->toContain('Stopped after reaching restart limit');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('uses a semantic dedupe key for restart limit notifications', function () {
|
|
||||||
$application = applicationWithRestartState();
|
|
||||||
$application->forceFill([
|
|
||||||
'name' => 'crashy-app',
|
|
||||||
'uuid' => 'application-uuid',
|
|
||||||
]);
|
|
||||||
$application->setRelation('environment', (object) [
|
|
||||||
'uuid' => 'environment-uuid',
|
|
||||||
'name' => 'production',
|
|
||||||
'project' => (object) ['uuid' => 'project-uuid'],
|
|
||||||
]);
|
|
||||||
|
|
||||||
$notification = new RestartLimitReached($application);
|
|
||||||
|
|
||||||
expect($notification->deduplicationKey((object) ['id' => 1], 'mail'))->toBe('restart-limit-reached:application:application-uuid:count:2')
|
|
||||||
->and($notification->deduplicateFor())->toBe(86400);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('keeps restart tracking configurable when stopping an application', function () {
|
it('keeps restart tracking configurable when stopping an application', function () {
|
||||||
$method = new ReflectionMethod(StopApplication::class, 'handle');
|
$method = new ReflectionMethod(StopApplication::class, 'handle');
|
||||||
$resetRestartCount = collect($method->getParameters())->firstWhere('name', 'resetRestartCount');
|
$resetRestartCount = collect($method->getParameters())->firstWhere('name', 'resetRestartCount');
|
||||||
|
|
|
||||||
|
|
@ -1,75 +0,0 @@
|
||||||
<?php
|
|
||||||
|
|
||||||
use App\Notifications\CustomEmailNotification;
|
|
||||||
use App\Services\NotificationDeduplicator;
|
|
||||||
use Illuminate\Notifications\Messages\MailMessage;
|
|
||||||
use Illuminate\Support\Facades\Cache;
|
|
||||||
|
|
||||||
class DedupeTestNotification extends CustomEmailNotification
|
|
||||||
{
|
|
||||||
public function __construct(
|
|
||||||
public ?string $semanticKey = null,
|
|
||||||
public bool $deduplicate = true,
|
|
||||||
public int $ttl = 900,
|
|
||||||
) {}
|
|
||||||
|
|
||||||
public function toMail(): MailMessage
|
|
||||||
{
|
|
||||||
return (new MailMessage)->subject('Test');
|
|
||||||
}
|
|
||||||
|
|
||||||
public function shouldDeduplicate(): bool
|
|
||||||
{
|
|
||||||
return $this->deduplicate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicateFor(): int
|
|
||||||
{
|
|
||||||
return $this->ttl;
|
|
||||||
}
|
|
||||||
|
|
||||||
public function deduplicationKey(object $notifiable, string $channel): ?string
|
|
||||||
{
|
|
||||||
return $this->semanticKey;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
beforeEach(function () {
|
|
||||||
Cache::flush();
|
|
||||||
$this->deduplicator = app(NotificationDeduplicator::class);
|
|
||||||
$this->notifiable = new class
|
|
||||||
{
|
|
||||||
public int $id = 123;
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
it('allows only the first identical notification fingerprint during the ttl', function () {
|
|
||||||
$notification = new DedupeTestNotification;
|
|
||||||
|
|
||||||
expect($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Body</p>'))->toBeTrue()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Body</p>'))->toBeFalse();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('allows different recipients and content through the default fingerprint', function () {
|
|
||||||
$notification = new DedupeTestNotification;
|
|
||||||
|
|
||||||
expect($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Body</p>'))->toBeTrue()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['second@example.com'], 'Subject', '<p>Body</p>'))->toBeTrue()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Other subject', '<p>Body</p>'))->toBeTrue()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Other body</p>'))->toBeTrue();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('uses semantic keys instead of rendered content when provided', function () {
|
|
||||||
$notification = new DedupeTestNotification(semanticKey: 'event:123');
|
|
||||||
|
|
||||||
expect($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Body</p>'))->toBeTrue()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Other subject', '<p>Other body</p>'))->toBeFalse()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['second@example.com'], 'Other subject', '<p>Other body</p>'))->toBeTrue();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('allows notifications to opt out of deduplication', function () {
|
|
||||||
$notification = new DedupeTestNotification(deduplicate: false);
|
|
||||||
|
|
||||||
expect($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Body</p>'))->toBeTrue()
|
|
||||||
->and($this->deduplicator->shouldSend($this->notifiable, $notification, 'mail', ['first@example.com'], 'Subject', '<p>Body</p>'))->toBeTrue();
|
|
||||||
});
|
|
||||||
Loading…
Reference in a new issue