Added dynamic column creator for diagnostic columns, added csv generator

diagnostic_csv.py

+import pandas as pd
+from sqlalchemy import func
+from sqlalchemy.orm import defer, Load
+
+import argparse
+
+import models
+
+
+def main():
+	session = models.get_session()
+	missions = session.query(models.missions).all()
+	for m in missions:
+		models.refresh()
+		q = session.query(
+				models.GliderLogs,
+				models.Surfacings,
+				models.Telemetry,
+				models.EWO,
+				models.Diagnostic
+			).join(
+				models.Diagnostic
+			).join(
+				models.Surfacings
+			).join(
+				models.Telemetry
+			).join(
+				models.EWO
+			).filter(
+				models.GliderLogs.missionID == m.id
+			)
+		q = q.options(
+			Load(models.GliderLogs).defer('id'),
+			Load(models.Surfacings).defer('id'),
+			Load(models.Surfacings).defer('missionID'),
+			Load(models.Telemetry).defer('id'),
+			Load(models.Telemetry).defer('missionID'),
+			Load(models.Telemetry).defer('gliderLogID'),
+			Load(models.Telemetry).defer('datetime'),
+			Load(models.EWO).defer('id'),
+			Load(models.EWO).defer('missionID'),
+			Load(models.EWO).defer('gliderLogID'),
+			Load(models.EWO).defer('datetime'),
+			Load(models.Diagnostic).defer('id'),
+			Load(models.Diagnostic).defer('missionID'),
+			Load(models.Diagnostic).defer('gliderLogID'),
+			Load(models.Diagnostic).defer('datetime'),
+		)
+		df = pd.read_sql(q.statement, q.session.bind)
+		glider = session.query(models.gliders).filter(
+			models.gliders.columns.id == m.gliderID
+		).one()
+		out_filename = 'm%s_%s_%s.csv' % (
+			                m.id,
+			                glider.name,
+			                str(m.startTime).replace(':', '_').replace(' ', '-')
+		                )
+		df.to_csv(out_filename, index=False)
+	return
+
+if __name__ == '__main__':
+	main()

models.py

 """A set of database models that will be used to store live slocum data"""
 import os
 
+import migrate.changeset
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Float, DateTime, ForeignKeyConstraint, ForeignKey, Enum, UniqueConstraint, Boolean
 from sqlalchemy.orm import sessionmaker
 
+
 import atexit
 
 import config
@@ -120,7 +122,7 @@ ewo = Table('ewo', m,
 
 
 m.create_all(engine)
-m.reflect(engine)
+m.reflect(bind=engine)
 Base = automap_base(metadata=m)
 Base.prepare()
 Session = sessionmaker(bind=engine)
@@ -134,6 +136,50 @@ Telemetry = Base.classes.telemetry
 Aborts = Base.classes.aborts
 EWO = Base.classes.ewo
 
+def refresh():
+    global m
+    global Base
+    global Session
+    global engine
+
+    global Gliders
+    global Missions
+    global Diagnostic
+    global Surfacings
+    global GliderLogs
+    global Telemetry
+    global Aborts
+    global EWO
+
+    m.reflect(bind=engine)
+    Base = automap_base(metadata=m)
+    Base.prepare(engine, reflect=True)
+    Session = sessionmaker(bind=engine)
+
+    Gliders = Base.classes.gliders
+    Missions = Base.classes.missions
+    Diagnostic = Base.classes.diagnostic
+    Surfacings = Base.classes.surfacings
+    GliderLogs = Base.classes.gliderLogs
+    Telemetry = Base.classes.telemetry
+    Aborts = Base.classes.aborts
+    EWO = Base.classes.ewo
+
+
+def update_diagnostic_columns(surface_message, already_defined):
+    m.bind = engine
+    for s in surface_message.sensors:
+        if s not in already_defined and not hasattr(Diagnostic, s):
+            print "Creating new diagnostic column: %s" % s
+            try:
+                float(surface_message.sensors[s].value)
+                col = Column(s, Float)
+            except ValueError as e:
+                col = Column(s, String)
+            col.create(Diagnostic.__table__)
+    refresh()
+    return
+
 
 def get_session():
     return Session()

parse_surfacelogs.py

@@ -217,8 +217,11 @@ def store_surfacelog(surface_log):
         mWaterVY=m_water_vy
     )
     session.add(telemetry_row)
+    session.commit()
 
     try:
+        already_defined = ['m_battery', 'm_leakdetect_voltage', 'm_vacuum', 'm_coulomb_amphr', 'm_coulomb_amphr_total', 'm_leakdetect_voltage_forward', 'm_water_vx', 'm_water_vy']
+        models.update_diagnostic_columns(surface_message, already_defined)
         diagnostic_row = models.Diagnostic(
             missionID=mission_id,
             gliderLogID=glider_log.id,
@@ -227,9 +230,15 @@ def store_surfacelog(surface_log):
             leakdetectVoltage=float(surface_message.sensors['m_leakdetect_voltage'].value),
             vacuum=float(surface_message.sensors['m_vacuum'].value),
             coulombAmphr=float(surface_message.sensors['m_coulomb_amphr'].value),
-            coulombAmphrTotal=float(surface_message.sensors['m_battery'].value),
+            coulombAmphrTotal=float(surface_message.sensors['m_coulomb_amphr_total'].value),
             leakdetectVoltageForward=float(surface_message.sensors['m_leakdetect_voltage_forward'].value)
         )
+        for s in surface_message.sensors:
+            if s not in already_defined and hasattr(diagnostic_row, s):
+                val = surface_message.sensors[s].value
+                if 'lat' in s or 'lon' in s:
+                    val = dinkum.webb_to_decdeg(float(val))
+                setattr(diagnostic_row, s, val)
         session.add(diagnostic_row)
     except KeyError:
         print "Incomplete logfile: %s Skipping." % surface_log.log_filename