Implement notification system that works even when database is unavailable. Problem: - When Uptime Kuma's database (external MariaDB/SQLite) becomes unavailable, UK stops functioning and cannot send notifications about its own failure. - Users are not alerted when UK itself is having database connectivity issues. Solution: - Added notification cache system that stores all active notification configs in memory when database is available - Implemented database error detection in both monitor.js and server.js error handlers to catch EHOSTUNREACH, ECONNREFUSED, and other DB errors - Added sendDatabaseDownNotification() method that uses cached notifications to send alerts when database connection fails - Cache automatically refreshes periodically (every 30 minutes) and when notifications are added/updated/deleted - Prevents duplicate notifications for the same database down event Changes: - server/notification.js: Added cache system and database down notification - server/server.js: Enhanced error handler to detect DB errors and trigger notifications, refresh cache on startup - server/model/monitor.js: Added DB error detection in safeBeat error handler - server/jobs.js: Added periodic cache refresh job (every 30 minutes) - test/backend-test/test-database-down-notification.js: Comprehensive test suite covering cache, notifications, error handling This ensures users are always notified when UK itself is having database connectivity issues, even if the database is completely unavailable.
443 lines
16 KiB
JavaScript
443 lines
16 KiB
JavaScript
const { R } = require("redbean-node");
|
|
const { log } = require("../src/util");
|
|
const Alerta = require("./notification-providers/alerta");
|
|
const AlertNow = require("./notification-providers/alertnow");
|
|
const AliyunSms = require("./notification-providers/aliyun-sms");
|
|
const Apprise = require("./notification-providers/apprise");
|
|
const Bale = require("./notification-providers/bale");
|
|
const Bark = require("./notification-providers/bark");
|
|
const Bitrix24 = require("./notification-providers/bitrix24");
|
|
const ClickSendSMS = require("./notification-providers/clicksendsms");
|
|
const CallMeBot = require("./notification-providers/call-me-bot");
|
|
const SMSC = require("./notification-providers/smsc");
|
|
const DingDing = require("./notification-providers/dingding");
|
|
const Discord = require("./notification-providers/discord");
|
|
const Elks = require("./notification-providers/46elks");
|
|
const Feishu = require("./notification-providers/feishu");
|
|
const Notifery = require("./notification-providers/notifery");
|
|
const FreeMobile = require("./notification-providers/freemobile");
|
|
const GoogleChat = require("./notification-providers/google-chat");
|
|
const Gorush = require("./notification-providers/gorush");
|
|
const Gotify = require("./notification-providers/gotify");
|
|
const GrafanaOncall = require("./notification-providers/grafana-oncall");
|
|
const HomeAssistant = require("./notification-providers/home-assistant");
|
|
const HeiiOnCall = require("./notification-providers/heii-oncall");
|
|
const Keep = require("./notification-providers/keep");
|
|
const Kook = require("./notification-providers/kook");
|
|
const Line = require("./notification-providers/line");
|
|
const LunaSea = require("./notification-providers/lunasea");
|
|
const Matrix = require("./notification-providers/matrix");
|
|
const Mattermost = require("./notification-providers/mattermost");
|
|
const NextcloudTalk = require("./notification-providers/nextcloudtalk");
|
|
const Nostr = require("./notification-providers/nostr");
|
|
const Ntfy = require("./notification-providers/ntfy");
|
|
const Octopush = require("./notification-providers/octopush");
|
|
const OneChat = require("./notification-providers/onechat");
|
|
const OneBot = require("./notification-providers/onebot");
|
|
const Opsgenie = require("./notification-providers/opsgenie");
|
|
const PagerDuty = require("./notification-providers/pagerduty");
|
|
const Pumble = require("./notification-providers/pumble");
|
|
const FlashDuty = require("./notification-providers/flashduty");
|
|
const PagerTree = require("./notification-providers/pagertree");
|
|
const PromoSMS = require("./notification-providers/promosms");
|
|
const Pushbullet = require("./notification-providers/pushbullet");
|
|
const PushDeer = require("./notification-providers/pushdeer");
|
|
const Pushover = require("./notification-providers/pushover");
|
|
const PushPlus = require("./notification-providers/pushplus");
|
|
const Pushy = require("./notification-providers/pushy");
|
|
const RocketChat = require("./notification-providers/rocket-chat");
|
|
const SerwerSMS = require("./notification-providers/serwersms");
|
|
const Signal = require("./notification-providers/signal");
|
|
const SIGNL4 = require("./notification-providers/signl4");
|
|
const Slack = require("./notification-providers/slack");
|
|
const SMSPartner = require("./notification-providers/smspartner");
|
|
const SMSEagle = require("./notification-providers/smseagle");
|
|
const SMTP = require("./notification-providers/smtp");
|
|
const Squadcast = require("./notification-providers/squadcast");
|
|
const Stackfield = require("./notification-providers/stackfield");
|
|
const Teams = require("./notification-providers/teams");
|
|
const TechulusPush = require("./notification-providers/techulus-push");
|
|
const Telegram = require("./notification-providers/telegram");
|
|
const Threema = require("./notification-providers/threema");
|
|
const Twilio = require("./notification-providers/twilio");
|
|
const Splunk = require("./notification-providers/splunk");
|
|
const Webhook = require("./notification-providers/webhook");
|
|
const WeCom = require("./notification-providers/wecom");
|
|
const GoAlert = require("./notification-providers/goalert");
|
|
const SMSManager = require("./notification-providers/smsmanager");
|
|
const ServerChan = require("./notification-providers/serverchan");
|
|
const ZohoCliq = require("./notification-providers/zoho-cliq");
|
|
const SevenIO = require("./notification-providers/sevenio");
|
|
const Whapi = require("./notification-providers/whapi");
|
|
const WAHA = require("./notification-providers/waha");
|
|
const Evolution = require("./notification-providers/evolution");
|
|
const GtxMessaging = require("./notification-providers/gtx-messaging");
|
|
const Cellsynt = require("./notification-providers/cellsynt");
|
|
const Onesender = require("./notification-providers/onesender");
|
|
const Wpush = require("./notification-providers/wpush");
|
|
const SendGrid = require("./notification-providers/send-grid");
|
|
const Brevo = require("./notification-providers/brevo");
|
|
const Resend = require("./notification-providers/resend");
|
|
const YZJ = require("./notification-providers/yzj");
|
|
const SMSPlanet = require("./notification-providers/sms-planet");
|
|
const SpugPush = require("./notification-providers/spugpush");
|
|
const SMSIR = require("./notification-providers/smsir");
|
|
const { commandExists } = require("./util-server");
|
|
const Webpush = require("./notification-providers/Webpush");
|
|
|
|
class Notification {
|
|
providerList = {};
|
|
/**
|
|
* Cache for all notifications to use when database is down
|
|
* @type {Array<object>}
|
|
*/
|
|
static notificationCache = [];
|
|
/**
|
|
* Last time the cache was refreshed
|
|
* @type {number}
|
|
*/
|
|
static cacheLastRefresh = 0;
|
|
/**
|
|
* Flag to track if we've already sent a database down notification
|
|
* @type {boolean}
|
|
*/
|
|
static databaseDownNotificationSent = false;
|
|
|
|
/**
|
|
* Initialize the notification providers
|
|
* @returns {void}
|
|
* @throws Notification provider does not have a name
|
|
* @throws Duplicate notification providers in list
|
|
*/
|
|
static init() {
|
|
log.debug("notification", "Prepare Notification Providers");
|
|
|
|
this.providerList = {};
|
|
|
|
const list = [
|
|
new Alerta(),
|
|
new AlertNow(),
|
|
new AliyunSms(),
|
|
new Apprise(),
|
|
new Bale(),
|
|
new Bark(),
|
|
new Bitrix24(),
|
|
new ClickSendSMS(),
|
|
new CallMeBot(),
|
|
new SMSC(),
|
|
new DingDing(),
|
|
new Discord(),
|
|
new Elks(),
|
|
new Feishu(),
|
|
new FreeMobile(),
|
|
new GoogleChat(),
|
|
new Gorush(),
|
|
new Gotify(),
|
|
new GrafanaOncall(),
|
|
new HomeAssistant(),
|
|
new HeiiOnCall(),
|
|
new Keep(),
|
|
new Kook(),
|
|
new Line(),
|
|
new LunaSea(),
|
|
new Matrix(),
|
|
new Mattermost(),
|
|
new NextcloudTalk(),
|
|
new Nostr(),
|
|
new Ntfy(),
|
|
new Octopush(),
|
|
new OneChat(),
|
|
new OneBot(),
|
|
new Onesender(),
|
|
new Opsgenie(),
|
|
new PagerDuty(),
|
|
new FlashDuty(),
|
|
new PagerTree(),
|
|
new PromoSMS(),
|
|
new Pumble(),
|
|
new Pushbullet(),
|
|
new PushDeer(),
|
|
new Pushover(),
|
|
new PushPlus(),
|
|
new Pushy(),
|
|
new RocketChat(),
|
|
new ServerChan(),
|
|
new SerwerSMS(),
|
|
new Signal(),
|
|
new SIGNL4(),
|
|
new SMSManager(),
|
|
new SMSPartner(),
|
|
new Slack(),
|
|
new SMSEagle(),
|
|
new SMTP(),
|
|
new Squadcast(),
|
|
new Stackfield(),
|
|
new Teams(),
|
|
new TechulusPush(),
|
|
new Telegram(),
|
|
new Threema(),
|
|
new Twilio(),
|
|
new Splunk(),
|
|
new Webhook(),
|
|
new WeCom(),
|
|
new GoAlert(),
|
|
new ZohoCliq(),
|
|
new SevenIO(),
|
|
new Whapi(),
|
|
new WAHA(),
|
|
new Evolution(),
|
|
new GtxMessaging(),
|
|
new Cellsynt(),
|
|
new Wpush(),
|
|
new Brevo(),
|
|
new Resend(),
|
|
new YZJ(),
|
|
new SMSPlanet(),
|
|
new SpugPush(),
|
|
new Notifery(),
|
|
new SMSIR(),
|
|
new SendGrid(),
|
|
new Webpush(),
|
|
];
|
|
for (let item of list) {
|
|
if (!item.name) {
|
|
throw new Error("Notification provider without name");
|
|
}
|
|
|
|
if (this.providerList[item.name]) {
|
|
throw new Error("Duplicate notification provider name");
|
|
}
|
|
this.providerList[item.name] = item;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send a notification
|
|
* @param {BeanModel} notification Notification to send
|
|
* @param {string} msg General Message
|
|
* @param {object} monitorJSON Monitor details (For Up/Down only)
|
|
* @param {object} heartbeatJSON Heartbeat details (For Up/Down only)
|
|
* @returns {Promise<string>} Successful msg
|
|
* @throws Error with fail msg
|
|
*/
|
|
static async send(
|
|
notification,
|
|
msg,
|
|
monitorJSON = null,
|
|
heartbeatJSON = null
|
|
) {
|
|
if (this.providerList[notification.type]) {
|
|
return this.providerList[notification.type].send(
|
|
notification,
|
|
msg,
|
|
monitorJSON,
|
|
heartbeatJSON
|
|
);
|
|
} else {
|
|
throw new Error("Notification type is not supported");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save a notification
|
|
* @param {object} notification Notification to save
|
|
* @param {?number} notificationID ID of notification to update
|
|
* @param {number} userID ID of user who adds notification
|
|
* @returns {Promise<Bean>} Notification that was saved
|
|
*/
|
|
static async save(notification, notificationID, userID) {
|
|
let bean;
|
|
|
|
if (notificationID) {
|
|
bean = await R.findOne("notification", " id = ? AND user_id = ? ", [
|
|
notificationID,
|
|
userID,
|
|
]);
|
|
|
|
if (!bean) {
|
|
throw new Error("notification not found");
|
|
}
|
|
} else {
|
|
bean = R.dispense("notification");
|
|
}
|
|
|
|
bean.name = notification.name;
|
|
bean.user_id = userID;
|
|
bean.config = JSON.stringify(notification);
|
|
bean.is_default = notification.isDefault || false;
|
|
await R.store(bean);
|
|
|
|
if (notification.applyExisting) {
|
|
await applyNotificationEveryMonitor(bean.id, userID);
|
|
}
|
|
|
|
// Refresh cache after saving
|
|
try {
|
|
await Notification.refreshCache();
|
|
} catch (e) {
|
|
// Silently fail - cache refresh is not critical
|
|
}
|
|
|
|
return bean;
|
|
}
|
|
|
|
/**
|
|
* Delete a notification
|
|
* @param {number} notificationID ID of notification to delete
|
|
* @param {number} userID ID of user who created notification
|
|
* @returns {Promise<void>}
|
|
*/
|
|
static async delete(notificationID, userID) {
|
|
let bean = await R.findOne("notification", " id = ? AND user_id = ? ", [
|
|
notificationID,
|
|
userID,
|
|
]);
|
|
|
|
if (!bean) {
|
|
throw new Error("notification not found");
|
|
}
|
|
|
|
await R.trash(bean);
|
|
|
|
// Refresh cache after deleting
|
|
try {
|
|
await Notification.refreshCache();
|
|
} catch (e) {
|
|
// Silently fail - cache refresh is not critical
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Check if apprise exists
|
|
* @returns {Promise<boolean>} Does the command apprise exist?
|
|
*/
|
|
static async checkApprise() {
|
|
return await commandExists("apprise");
|
|
}
|
|
|
|
/**
|
|
* Load all notifications into cache for use when database is down
|
|
* @returns {Promise<void>}
|
|
*/
|
|
static async refreshCache() {
|
|
try {
|
|
// Get all notifications (including default ones)
|
|
const notifications = await R.getAll(`
|
|
SELECT notification.*
|
|
FROM notification
|
|
WHERE active = 1
|
|
`);
|
|
|
|
this.notificationCache = notifications.map(bean => {
|
|
try {
|
|
const config = JSON.parse(bean.config || "{}");
|
|
return {
|
|
id: bean.id,
|
|
name: bean.name,
|
|
type: config.type,
|
|
config: config,
|
|
is_default: bean.is_default === 1,
|
|
user_id: bean.user_id,
|
|
};
|
|
} catch (e) {
|
|
log.warn("notification", `Failed to parse notification config for ${bean.id}: ${e.message}`);
|
|
return null;
|
|
}
|
|
}).filter(n => n !== null);
|
|
|
|
this.cacheLastRefresh = Date.now();
|
|
log.debug("notification", `Refreshed notification cache with ${this.notificationCache.length} notifications`);
|
|
} catch (e) {
|
|
log.error("notification", `Failed to refresh notification cache: ${e.message}`);
|
|
// Don't clear the cache if refresh fails, keep using old cache
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Send notification about database being down using cached notifications
|
|
* @param {string} errorMessage Error message from database connection failure
|
|
* @returns {Promise<void>}
|
|
*/
|
|
static async sendDatabaseDownNotification(errorMessage) {
|
|
// Only send once per database down event
|
|
if (this.databaseDownNotificationSent) {
|
|
return;
|
|
}
|
|
|
|
// Check if cache is empty or too old (older than 1 hour)
|
|
const cacheAge = Date.now() - this.cacheLastRefresh;
|
|
if (this.notificationCache.length === 0 || cacheAge > 60 * 60 * 1000) {
|
|
log.warn("notification", "Notification cache is empty or too old, cannot send database down notification");
|
|
return;
|
|
}
|
|
|
|
this.databaseDownNotificationSent = true;
|
|
|
|
const msg = `🔴 Uptime Kuma Database Connection Failed\n\nError: ${errorMessage}\n\nUptime Kuma is unable to connect to its database. Monitoring may be affected.`;
|
|
|
|
// Send to all cached notifications
|
|
for (const notification of this.notificationCache) {
|
|
try {
|
|
await this.send(
|
|
notification.config,
|
|
msg,
|
|
{
|
|
id: 0,
|
|
name: "Uptime Kuma System",
|
|
type: "system",
|
|
active: true,
|
|
},
|
|
{
|
|
status: "down",
|
|
msg: errorMessage,
|
|
time: new Date().toISOString(),
|
|
ping: null,
|
|
}
|
|
);
|
|
log.info("notification", `Sent database down notification via ${notification.name} (${notification.type})`);
|
|
} catch (e) {
|
|
log.error("notification", `Failed to send database down notification via ${notification.name}: ${e.message}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Reset the database down notification flag (call when database is back up)
|
|
* @returns {void}
|
|
*/
|
|
static resetDatabaseDownFlag() {
|
|
this.databaseDownNotificationSent = false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Apply the notification to every monitor
|
|
* @param {number} notificationID ID of notification to apply
|
|
* @param {number} userID ID of user who created notification
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async function applyNotificationEveryMonitor(notificationID, userID) {
|
|
let monitors = await R.getAll("SELECT id FROM monitor WHERE user_id = ?", [
|
|
userID,
|
|
]);
|
|
|
|
for (let i = 0; i < monitors.length; i++) {
|
|
let checkNotification = await R.findOne(
|
|
"monitor_notification",
|
|
" monitor_id = ? AND notification_id = ? ",
|
|
[ monitors[i].id, notificationID ]
|
|
);
|
|
|
|
if (!checkNotification) {
|
|
let relation = R.dispense("monitor_notification");
|
|
relation.monitor_id = monitors[i].id;
|
|
relation.notification_id = notificationID;
|
|
await R.store(relation);
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
Notification,
|
|
};
|