move critical email errors to gmail logging handler
This commit is contained in:
petrucci4prez
parent
af32fc6d26
commit
f64e01f896
|
|
@ -1,3 +1,6 @@
|
|||
recipientList:
|
||||
- natedwarshuis@gmail.com
|
||||
gmail:
|
||||
username: natedwarshuis@gmail.com
|
||||
passwd: bwsasfxqjbookmed
|
||||
recipientList:
|
||||
- natedwarshuis@gmail.com
|
||||
state: disarmed
|
||||
|
|
|
|||
11
main.py
11
main.py
|
|
@ -35,8 +35,6 @@ if __name__ == '__main__':
|
|||
GPIO.setmode(GPIO.BCM)
|
||||
resetUSBDevice('1-1')
|
||||
|
||||
from notifier import criticalError
|
||||
|
||||
stateMachine = StateMachine()
|
||||
|
||||
# TODO: segfaults are annoying :(
|
||||
|
|
@ -47,14 +45,7 @@ if __name__ == '__main__':
|
|||
time.sleep(31536000)
|
||||
|
||||
except Exception:
|
||||
t = traceback.format_exc()
|
||||
|
||||
try:
|
||||
criticalError(t)
|
||||
except NameError:
|
||||
pass
|
||||
|
||||
logger.critical(t)
|
||||
logger.critical(traceback.format_exc())
|
||||
|
||||
finally:
|
||||
clean()
|
||||
|
|
|
|||
62
notifier.py
62
notifier.py
|
|
@ -5,60 +5,70 @@ from datetime import datetime
|
|||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
COMMASPACE=', '
|
||||
gmail = ConfigFile('config.yaml')['gmail']
|
||||
|
||||
RECIPIENT_LIST=ConfigFile('config.yaml')['recipientList']
|
||||
|
||||
GMAIL_USER='natedwarshuis@gmail.com'
|
||||
GMAIL_PWD='bwsasfxqjbookmed'
|
||||
|
||||
def getNextDate():
|
||||
def _getNextDate():
|
||||
m = datetime.now().month + 1
|
||||
y = datetime.now().year
|
||||
y = y + 1 if m > 12 else y
|
||||
return datetime(year=y, month=m%12, day=1, hour=12, minute=0)
|
||||
|
||||
@async(daemon=True)
|
||||
def scheduleAction(action):
|
||||
def _scheduleAction(action):
|
||||
while 1:
|
||||
nextDate = getNextDate()
|
||||
nextDate = _getNextDate()
|
||||
sleepTime = nextDate - datetime.today()
|
||||
logger.info('Next monthly test scheduled at %s (%s)', nextDate, sleepTime)
|
||||
_logger.info('Next monthly test scheduled at %s (%s)', nextDate, sleepTime)
|
||||
time.sleep(sleepTime.days * 86400 + sleepTime.seconds)
|
||||
action()
|
||||
|
||||
# probably an easier way to do this in logging module
|
||||
@async(daemon=False)
|
||||
def sendEmail(subject, body):
|
||||
def _sendToGmail(username, passwd, recipiantList, subject, body, server='smtp.gmail.com', port=587):
|
||||
msg = MIMEMultipart()
|
||||
msg['Subject'] = subject
|
||||
msg['From'] = GMAIL_USER
|
||||
msg['To'] = COMMASPACE.join(RECIPIENT_LIST)
|
||||
msg['From'] = username
|
||||
msg['To'] = ', '.join(recipiantList)
|
||||
msg.attach(MIMEText(body, 'plain'))
|
||||
|
||||
s = SMTP('smtp.gmail.com', 587)
|
||||
s = SMTP(server, port)
|
||||
s.starttls()
|
||||
s.login(GMAIL_USER, GMAIL_PWD)
|
||||
s.login(username, passwd)
|
||||
s.send_message(msg)
|
||||
s.quit()
|
||||
|
||||
def monthlyTest():
|
||||
subject = 'harrison4hegemon - automated monthly test'
|
||||
body = 'this is an automated message - please do not reply\n\nin the future this may have useful information'
|
||||
sendEmail(subject, body)
|
||||
logger.info('Sending monthly test to email list')
|
||||
sendEmail(gmail['username'], gmail['passwd'], gmail['recipientList'], subject, body)
|
||||
_logger.debug('Sending monthly test to email list')
|
||||
|
||||
def intruderAlert():
|
||||
subject = 'harrison4hegemon - intruder detected'
|
||||
body = 'intruder detected - alarm was tripped on ' + time.strftime("%H:%M:%S - %d/%m/%Y")
|
||||
sendEmail(subject, body)
|
||||
logger.info('Sending intruder alert to email list')
|
||||
sendEmail(gmail['username'], gmail['passwd'], gmail['recipientList'], subject, body)
|
||||
_logger.info('intruder detected')
|
||||
_logger.debug('Sending intruder alert to email list')
|
||||
|
||||
def criticalError(err):
|
||||
subject = 'harrison4hegemon - critical error'
|
||||
sendEmail(subject, err)
|
||||
logger.info('Sending critical error to email list')
|
||||
class GmailHandler(logging.Handler):
|
||||
'''
|
||||
Logging handler that sends records to gmail. This is almost like the
|
||||
SMTPHandler except that the username and fromaddr are the same and
|
||||
credentials are mandatory
|
||||
'''
|
||||
def __init__(self, username, passwd, recipientList, subject):
|
||||
super().__init__()
|
||||
self.username = username
|
||||
self.passwd = passwd
|
||||
self.recipientList = recipientList
|
||||
self.subject = subject
|
||||
|
||||
scheduleAction(monthlyTest)
|
||||
def emit(self, record):
|
||||
try:
|
||||
_sendToGmail(self.username, self.passwd, self.recipientList,
|
||||
self.subject, self.format(record))
|
||||
except:
|
||||
self.handleError(record)
|
||||
|
||||
_scheduleAction(monthlyTest)
|
||||
|
|
|
|||
|
|
@ -1,33 +1,40 @@
|
|||
import logging, os
|
||||
from subprocess import run, PIPE, CalledProcessError
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
from logging.handlers import TimedRotatingFileHandler, SMTPHandler
|
||||
|
||||
"""
|
||||
'''
|
||||
Logger conventions
|
||||
- CRITICAL: for things that cause crashes. sends email
|
||||
- CRITICAL: for things that cause crashes. only level with gmail
|
||||
- ERROR: for things that cause startup/shutdown issues
|
||||
- WARNING: for recoverable issues that may cause future problems
|
||||
- INFO: state changes and sensor readings
|
||||
- DEBUG: all extraneous crap
|
||||
"""
|
||||
|
||||
# formats console output depending on whether we have gluster
|
||||
Init order (very essential)
|
||||
1) init console output (this will go to journald) and format as console only
|
||||
2) init the module level _logger so we can log anything that happens as we build
|
||||
the other loggers
|
||||
3) mount glusterfs, any errors here will go to console output
|
||||
4) once gluster is mounted, add to root _logger and remove "console only" warning
|
||||
from console
|
||||
5) import gmail, this must come here as it uses loggers for some of its setup
|
||||
6) init gmail handler
|
||||
'''
|
||||
|
||||
def _formatConsole(gluster = False):
|
||||
'''
|
||||
formats console output depending on whether we have gluster
|
||||
'''
|
||||
c = '' if gluster else '[CONSOLE ONLY] '
|
||||
fmt = logging.Formatter('[%(name)s] [%(levelname)s] ' + c + '%(message)s')
|
||||
console.setFormatter(fmt)
|
||||
|
||||
# init console, but don't expect gluster to be here yet
|
||||
console = logging.StreamHandler()
|
||||
_formatConsole(gluster = False)
|
||||
|
||||
rootLogger = logging.getLogger()
|
||||
rootLogger.setLevel(logging.DEBUG)
|
||||
rootLogger.addHandler(console)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class GlusterFSHandler(TimedRotatingFileHandler):
|
||||
'''
|
||||
Logic to mount timed rotating file within a gluster volume. Note that this
|
||||
class will mount itself automatically. Note that the actual filepaths for
|
||||
logging are hardcoded here
|
||||
'''
|
||||
def __init__(self, server, volume, mountpoint, options=None):
|
||||
if not os.path.exists(mountpoint):
|
||||
raise FileNotFoundError
|
||||
|
|
@ -42,7 +49,7 @@ class GlusterFSHandler(TimedRotatingFileHandler):
|
|||
if not os.path.exists(logdest):
|
||||
os.mkdir(logdest)
|
||||
elif os.path.isfile(logdest):
|
||||
logger.error('%s is present but is a file (vs a directory). ' \
|
||||
_logger.error('%s is present but is a file (vs a directory). ' \
|
||||
'Please (re)move this file to prevent data loss', logdest)
|
||||
raise SystemExit
|
||||
|
||||
|
|
@ -55,8 +62,8 @@ class GlusterFSHandler(TimedRotatingFileHandler):
|
|||
|
||||
def _mount(self):
|
||||
if os.path.ismount(self._mountpoint):
|
||||
# NOTE: this assumes that the already-mounted device is the one intended
|
||||
logger.warning('Device already mounted at {}'.format(self._mountpoint))
|
||||
# this assumes that the already-mounted device is the one intended
|
||||
_logger.warning('Device already mounted at {}'.format(self._mountpoint))
|
||||
else:
|
||||
dst = self._server + ':/' + self._volume
|
||||
cmd = ['mount', '-t', 'glusterfs', dst, self._mountpoint]
|
||||
|
|
@ -72,14 +79,32 @@ class GlusterFSHandler(TimedRotatingFileHandler):
|
|||
run(cmd, check=True, stdout=PIPE, stderr=PIPE)
|
||||
except CalledProcessError as e:
|
||||
stderr = e.stderr.decode('ascii').rstrip()
|
||||
logger.error(stderr)
|
||||
_logger.error(stderr)
|
||||
raise SystemExit
|
||||
|
||||
def close(self):
|
||||
TimedRotatingFileHandler.close(self) # must close file stream before unmounting
|
||||
'''
|
||||
Close file and dismount (must be in this order). Called when
|
||||
'removeHandler' is invoked
|
||||
'''
|
||||
TimedRotatingFileHandler.close(self)
|
||||
self._unmount()
|
||||
|
||||
# ...now activate gluster
|
||||
'''
|
||||
Init sequence (see above)
|
||||
'''
|
||||
# 1
|
||||
console = logging.StreamHandler()
|
||||
_formatConsole(gluster = False)
|
||||
|
||||
rootLogger = logging.getLogger()
|
||||
rootLogger.setLevel(logging.DEBUG)
|
||||
rootLogger.addHandler(console)
|
||||
|
||||
# 2
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
# 3
|
||||
gluster = GlusterFSHandler(
|
||||
server = '192.168.11.39',
|
||||
volume = 'pyledriver',
|
||||
|
|
@ -87,10 +112,22 @@ gluster = GlusterFSHandler(
|
|||
options = 'backupvolfile-server=192.168.11.48'
|
||||
)
|
||||
|
||||
# 4
|
||||
_formatConsole(gluster = True)
|
||||
rootLogger.addHandler(gluster)
|
||||
|
||||
# this should only be called at the end to clean up
|
||||
# 5
|
||||
from notifier import gmail, GmailHandler
|
||||
|
||||
# 6
|
||||
gmail = GmailHandler(gmail['username'], gmail['passwd'], gmail['recipientList'],
|
||||
'harrison4hegemon - critical error')
|
||||
gmail.setLevel(logging.CRITICAL)
|
||||
rootLogger.addHandler(gmail)
|
||||
|
||||
'''
|
||||
Clean up
|
||||
'''
|
||||
def unmountGluster():
|
||||
rootLogger.removeHandler(gluster)
|
||||
_formatConsole(gluster = False)
|
||||
|
|
|
|||
Loading…
Reference in New Issue