ndwarshuis
/
pyledriver
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

move critical email errors to gmail logging handler

This commit is contained in:
petrucci4prez 2017-05-31 00:21:23 -04:00
parent af32fc6d26
commit f64e01f896
4 changed files with 102 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
config.yaml
View File

@ -1,3 +1,6 @@
recipientList:
- natedwarshuis@gmail.com
gmail:
username: natedwarshuis@gmail.com
passwd: bwsasfxqjbookmed
recipientList:
- natedwarshuis@gmail.com
state: disarmed

11
main.py
View File

@ -35,8 +35,6 @@ if __name__ == '__main__':
GPIO.setmode(GPIO.BCM)
resetUSBDevice('1-1')
from notifier import criticalError
stateMachine = StateMachine()
# TODO: segfaults are annoying :(
@ -47,14 +45,7 @@ if __name__ == '__main__':
time.sleep(31536000)
except Exception:
t = traceback.format_exc()
try:
criticalError(t)
except NameError:
pass
logger.critical(t)
logger.critical(traceback.format_exc())
finally:
clean()

64
notifier.py
View File

@ -5,60 +5,70 @@ from datetime import datetime
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
logger = logging.getLogger(__name__)
_logger = logging.getLogger(__name__)
COMMASPACE=', '
gmail = ConfigFile('config.yaml')['gmail']
RECIPIENT_LIST=ConfigFile('config.yaml')['recipientList']
GMAIL_USER='natedwarshuis@gmail.com'
GMAIL_PWD='bwsasfxqjbookmed'
def getNextDate():
def _getNextDate():
m = datetime.now().month + 1
y = datetime.now().year
y = y + 1 if m > 12 else y
return datetime(year=y, month=m%12, day=1, hour=12, minute=0)
@async(daemon=True)
def scheduleAction(action):
def _scheduleAction(action):
while 1:
nextDate = getNextDate()
nextDate = _getNextDate()
sleepTime = nextDate - datetime.today()
logger.info('Next monthly test scheduled at %s (%s)', nextDate, sleepTime)
_logger.info('Next monthly test scheduled at %s (%s)', nextDate, sleepTime)
time.sleep(sleepTime.days * 86400 + sleepTime.seconds)
action()
# probably an easier way to do this in logging module
@async(daemon=False)
def sendEmail(subject, body):
def _sendToGmail(username, passwd, recipiantList, subject, body, server='smtp.gmail.com', port=587):
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = GMAIL_USER
msg['To'] = COMMASPACE.join(RECIPIENT_LIST)
msg['From'] = username
msg['To'] = ', '.join(recipiantList)
msg.attach(MIMEText(body, 'plain'))
s = SMTP('smtp.gmail.com', 587)
s = SMTP(server, port)
s.starttls()
s.login(GMAIL_USER, GMAIL_PWD)
s.login(username, passwd)
s.send_message(msg)
s.quit()
def monthlyTest():
subject = 'harrison4hegemon - automated monthly test'
body = 'this is an automated message - please do not reply\n\nin the future this may have useful information'
sendEmail(subject, body)
logger.info('Sending monthly test to email list')
sendEmail(gmail['username'], gmail['passwd'], gmail['recipientList'], subject, body)
_logger.debug('Sending monthly test to email list')
def intruderAlert():
subject = 'harrison4hegemon - intruder detected'
body = 'intruder detected - alarm was tripped on ' + time.strftime("%H:%M:%S - %d/%m/%Y")
sendEmail(subject, body)
logger.info('Sending intruder alert to email list')
def criticalError(err):
subject = 'harrison4hegemon - critical error'
sendEmail(subject, err)
logger.info('Sending critical error to email list')
sendEmail(gmail['username'], gmail['passwd'], gmail['recipientList'], subject, body)
_logger.info('intruder detected')
_logger.debug('Sending intruder alert to email list')
scheduleAction(monthlyTest)
class GmailHandler(logging.Handler):
'''
Logging handler that sends records to gmail. This is almost like the
SMTPHandler except that the username and fromaddr are the same and
credentials are mandatory
'''
def __init__(self, username, passwd, recipientList, subject):
super().__init__()
self.username = username
self.passwd = passwd
self.recipientList = recipientList
self.subject = subject
def emit(self, record):
try:
_sendToGmail(self.username, self.passwd, self.recipientList,
self.subject, self.format(record))
except:
self.handleError(record)
_scheduleAction(monthlyTest)

81
sharedLogging.py
View File

@ -1,33 +1,40 @@
import logging, os
from subprocess import run, PIPE, CalledProcessError
from logging.handlers import TimedRotatingFileHandler
from logging.handlers import TimedRotatingFileHandler, SMTPHandler
"""
'''
Logger conventions
- CRITICAL: for things that cause crashes. sends email
- CRITICAL: for things that cause crashes. only level with gmail
- ERROR: for things that cause startup/shutdown issues
- WARNING: for recoverable issues that may cause future problems
- INFO: state changes and sensor readings
- DEBUG: all extraneous crap
"""
# formats console output depending on whether we have gluster
Init order (very essential)
1) init console output (this will go to journald) and format as console only
2) init the module level _logger so we can log anything that happens as we build
the other loggers
3) mount glusterfs, any errors here will go to console output
4) once gluster is mounted, add to root _logger and remove "console only" warning
from console
5) import gmail, this must come here as it uses loggers for some of its setup
6) init gmail handler
'''
def _formatConsole(gluster = False):
'''
formats console output depending on whether we have gluster
'''
c = '' if gluster else '[CONSOLE ONLY] '
fmt = logging.Formatter('[%(name)s] [%(levelname)s] ' + c + '%(message)s')
console.setFormatter(fmt)
# init console, but don't expect gluster to be here yet
console = logging.StreamHandler()
_formatConsole(gluster = False)
rootLogger = logging.getLogger()
rootLogger.setLevel(logging.DEBUG)
rootLogger.addHandler(console)
logger = logging.getLogger(__name__)
class GlusterFSHandler(TimedRotatingFileHandler):
'''
Logic to mount timed rotating file within a gluster volume. Note that this
class will mount itself automatically. Note that the actual filepaths for
logging are hardcoded here
'''
def __init__(self, server, volume, mountpoint, options=None):
if not os.path.exists(mountpoint):
raise FileNotFoundError
@ -42,7 +49,7 @@ class GlusterFSHandler(TimedRotatingFileHandler):
if not os.path.exists(logdest):
os.mkdir(logdest)
elif os.path.isfile(logdest):
logger.error('%s is present but is a file (vs a directory). ' \
_logger.error('%s is present but is a file (vs a directory). ' \
'Please (re)move this file to prevent data loss', logdest)
raise SystemExit
@ -55,8 +62,8 @@ class GlusterFSHandler(TimedRotatingFileHandler):
def _mount(self):
if os.path.ismount(self._mountpoint):
# NOTE: this assumes that the already-mounted device is the one intended
logger.warning('Device already mounted at {}'.format(self._mountpoint))
# this assumes that the already-mounted device is the one intended
_logger.warning('Device already mounted at {}'.format(self._mountpoint))
else:
dst = self._server + ':/' + self._volume
cmd = ['mount', '-t', 'glusterfs', dst, self._mountpoint]
@ -72,14 +79,32 @@ class GlusterFSHandler(TimedRotatingFileHandler):
run(cmd, check=True, stdout=PIPE, stderr=PIPE)
except CalledProcessError as e:
stderr = e.stderr.decode('ascii').rstrip()
logger.error(stderr)
_logger.error(stderr)
raise SystemExit
def close(self):
TimedRotatingFileHandler.close(self) # must close file stream before unmounting
'''
Close file and dismount (must be in this order). Called when
'removeHandler' is invoked
'''
TimedRotatingFileHandler.close(self)
self._unmount()
# ...now activate gluster
'''
Init sequence (see above)
'''
# 1
console = logging.StreamHandler()
_formatConsole(gluster = False)
rootLogger = logging.getLogger()
rootLogger.setLevel(logging.DEBUG)
rootLogger.addHandler(console)
# 2
_logger = logging.getLogger(__name__)
# 3
gluster = GlusterFSHandler(
server = '192.168.11.39',
volume = 'pyledriver',
@ -87,10 +112,22 @@ gluster = GlusterFSHandler(
options = 'backupvolfile-server=192.168.11.48'
)
# 4
_formatConsole(gluster = True)
rootLogger.addHandler(gluster)
# this should only be called at the end to clean up
# 5
from notifier import gmail, GmailHandler
# 6
gmail = GmailHandler(gmail['username'], gmail['passwd'], gmail['recipientList'],
'harrison4hegemon - critical error')
gmail.setLevel(logging.CRITICAL)
rootLogger.addHandler(gmail)
'''
Clean up
'''
def unmountGluster():
rootLogger.removeHandler(gluster)
_formatConsole(gluster = False)