move critical email errors to gmail logging handler

config.yaml

@@ -1,3 +1,6 @@
-recipientList:
+gmail:
-- natedwarshuis@gmail.com
+  username: natedwarshuis@gmail.com
+  passwd: bwsasfxqjbookmed
+  recipientList:
+  - natedwarshuis@gmail.com
 state: disarmed

main.py

@@ -35,8 +35,6 @@ if __name__ == '__main__':
 		GPIO.setmode(GPIO.BCM)
 		resetUSBDevice('1-1')
 
-		from notifier import criticalError
-		
 		stateMachine = StateMachine()
 
 		# TODO: segfaults are annoying :(
@@ -47,14 +45,7 @@ if __name__ == '__main__':
 			time.sleep(31536000)
 
 	except Exception:
-		t = traceback.format_exc()
+		logger.critical(traceback.format_exc())
-
-		try:
-			criticalError(t)
-		except NameError:
-			pass
-			
-		logger.critical(t)
 	
 	finally:
 		clean()

notifier.py

@@ -5,60 +5,70 @@ from datetime import datetime
 from email.mime.multipart import MIMEMultipart
 from email.mime.text  import MIMEText 
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
-COMMASPACE=', '
+gmail = ConfigFile('config.yaml')['gmail']
 
-RECIPIENT_LIST=ConfigFile('config.yaml')['recipientList']
+def _getNextDate():
-
-GMAIL_USER='natedwarshuis@gmail.com'
-GMAIL_PWD='bwsasfxqjbookmed'
-
-def getNextDate():
 	m = datetime.now().month + 1
 	y = datetime.now().year
 	y = y + 1 if m > 12 else y
 	return datetime(year=y, month=m%12, day=1, hour=12, minute=0)
 
 @async(daemon=True)
-def scheduleAction(action):
+def _scheduleAction(action):
 	while 1:
-		nextDate = getNextDate()
+		nextDate = _getNextDate()
 		sleepTime = nextDate - datetime.today()
-		logger.info('Next monthly test scheduled at %s (%s)', nextDate, sleepTime)
+		_logger.info('Next monthly test scheduled at %s (%s)', nextDate, sleepTime)
 		time.sleep(sleepTime.days * 86400 + sleepTime.seconds)
 		action()
 
-# probably an easier way to do this in logging module
 @async(daemon=False)
-def sendEmail(subject, body):
+def _sendToGmail(username, passwd, recipiantList, subject, body, server='smtp.gmail.com', port=587):
 	msg = MIMEMultipart()
 	msg['Subject'] =  subject
-	msg['From'] = GMAIL_USER
+	msg['From'] = username
-	msg['To'] = COMMASPACE.join(RECIPIENT_LIST)
+	msg['To'] = ', '.join(recipiantList)
 	msg.attach(MIMEText(body, 'plain'))
 
-	s = SMTP('smtp.gmail.com', 587)
+	s = SMTP(server, port)
 	s.starttls()
-	s.login(GMAIL_USER, GMAIL_PWD)
+	s.login(username, passwd)
 	s.send_message(msg)
 	s.quit()
 
 def monthlyTest():
 	subject = 'harrison4hegemon - automated monthly test'
 	body = 'this is an automated message - please do not reply\n\nin the future this may have useful information'
-	sendEmail(subject, body)
+	sendEmail(gmail['username'], gmail['passwd'], gmail['recipientList'], subject, body)
-	logger.info('Sending monthly test to email list')
+	_logger.debug('Sending monthly test to email list')
 	
 def intruderAlert():
 	subject = 'harrison4hegemon - intruder detected'
 	body = 'intruder detected - alarm was tripped on ' + time.strftime("%H:%M:%S - %d/%m/%Y")
-	sendEmail(subject, body)
+	sendEmail(gmail['username'], gmail['passwd'], gmail['recipientList'], subject, body)
-	logger.info('Sending intruder alert to email list')
+	_logger.info('intruder detected')
-	
+	_logger.debug('Sending intruder alert to email list')
-def criticalError(err):
-	subject = 'harrison4hegemon - critical error'
-	sendEmail(subject, err)
-	logger.info('Sending critical error to email list')
 
-scheduleAction(monthlyTest)
+class GmailHandler(logging.Handler):
+	'''
+	Logging handler that sends records to gmail. This is almost like the
+	SMTPHandler except that the username and fromaddr are the same and
+	credentials are mandatory
+	'''
+	def __init__(self, username, passwd, recipientList, subject):
+		super().__init__()
+		self.username = username
+		self.passwd = passwd
+		self.recipientList = recipientList
+		self.subject = subject
+		
+	def emit(self, record):
+		try:
+			_sendToGmail(self.username, self.passwd, self.recipientList,
+				self.subject, self.format(record))
+		except:
+			self.handleError(record)
+
+_scheduleAction(monthlyTest)

sharedLogging.py

@@ -1,33 +1,40 @@
 import logging, os
 from subprocess import run, PIPE, CalledProcessError
-from logging.handlers import TimedRotatingFileHandler
+from logging.handlers import TimedRotatingFileHandler, SMTPHandler
 
-"""
+'''
 Logger conventions
-- CRITICAL: for things that cause crashes. sends email
+- CRITICAL: for things that cause crashes. only level with gmail
 - ERROR: for things that cause startup/shutdown issues
 - WARNING: for recoverable issues that may cause future problems
 - INFO: state changes and sensor readings
 - DEBUG: all extraneous crap
-"""
 
-# formats console output depending on whether we have gluster
+Init order (very essential)
+1) init console output (this will go to journald) and format as console only
+2) init the module level _logger so we can log anything that happens as we build
+   the other loggers
+3) mount glusterfs, any errors here will go to console output
+4) once gluster is mounted, add to root _logger and remove "console only" warning
+   from console
+5) import gmail, this must come here as it uses loggers for some of its setup
+6) init gmail handler
+'''
+
 def _formatConsole(gluster = False):
+	'''
+	formats console output depending on whether we have gluster
+	'''
 	c = '' if gluster else '[CONSOLE ONLY] '
 	fmt = logging.Formatter('[%(name)s] [%(levelname)s] ' + c + '%(message)s')
 	console.setFormatter(fmt)	
 
-# init console, but don't expect gluster to be here yet
-console = logging.StreamHandler()
-_formatConsole(gluster = False)
-
-rootLogger = logging.getLogger()
-rootLogger.setLevel(logging.DEBUG)
-rootLogger.addHandler(console)
-
-logger = logging.getLogger(__name__)
-
 class GlusterFSHandler(TimedRotatingFileHandler):
+	'''
+	Logic to mount timed rotating file within a gluster volume. Note that this
+	class will mount itself automatically. Note that the actual filepaths for
+		logging are hardcoded here
+	'''
 	def __init__(self, server, volume, mountpoint, options=None):
 		if not os.path.exists(mountpoint):
 			raise FileNotFoundError
@@ -42,7 +49,7 @@ class GlusterFSHandler(TimedRotatingFileHandler):
 		if not os.path.exists(logdest):
 			os.mkdir(logdest)
 		elif os.path.isfile(logdest):
-			logger.error('%s is present but is a file (vs a directory). ' \
+			_logger.error('%s is present but is a file (vs a directory). ' \
 				'Please (re)move this file to prevent data loss', logdest)
 			raise SystemExit
 		
@@ -55,8 +62,8 @@ class GlusterFSHandler(TimedRotatingFileHandler):
 
 	def _mount(self):
 		if os.path.ismount(self._mountpoint):
-			# NOTE: this assumes that the already-mounted device is the one intended
+			# this assumes that the already-mounted device is the one intended
-			logger.warning('Device already mounted at {}'.format(self._mountpoint))
+			_logger.warning('Device already mounted at {}'.format(self._mountpoint))
 		else:
 			dst = self._server + ':/' + self._volume
 			cmd = ['mount', '-t', 'glusterfs', dst, self._mountpoint]
@@ -72,14 +79,32 @@ class GlusterFSHandler(TimedRotatingFileHandler):
 			run(cmd, check=True, stdout=PIPE, stderr=PIPE)
 		except CalledProcessError as e:
 			stderr = e.stderr.decode('ascii').rstrip()
-			logger.error(stderr)
+			_logger.error(stderr)
 			raise SystemExit
 			
 	def close(self):
-		TimedRotatingFileHandler.close(self) # must close file stream before unmounting
+		'''
+		Close file and dismount (must be in this order). Called when
+		'removeHandler' is invoked
+		'''
+		TimedRotatingFileHandler.close(self)
 		self._unmount()
 
-# ...now activate gluster
+'''
+Init sequence (see above)
+'''
+# 1
+console = logging.StreamHandler()
+_formatConsole(gluster = False)
+
+rootLogger = logging.getLogger()
+rootLogger.setLevel(logging.DEBUG)
+rootLogger.addHandler(console)
+
+# 2
+_logger = logging.getLogger(__name__)
+
+# 3
 gluster = GlusterFSHandler(
 	server = '192.168.11.39',
 	volume = 'pyledriver',
@@ -87,10 +112,22 @@ gluster = GlusterFSHandler(
 	options = 'backupvolfile-server=192.168.11.48'
 )
 
+# 4
 _formatConsole(gluster = True)
 rootLogger.addHandler(gluster)
 
-# this should only be called at the end to clean up
+# 5
+from notifier import gmail, GmailHandler
+
+# 6
+gmail = GmailHandler(gmail['username'], gmail['passwd'], gmail['recipientList'],
+	'harrison4hegemon - critical error')
+gmail.setLevel(logging.CRITICAL)
+rootLogger.addHandler(gmail)
+
+'''
+Clean up
+'''
 def unmountGluster():
 	rootLogger.removeHandler(gluster)
 	_formatConsole(gluster = False)