Commit 4a1fe382 authored by Brad Covey's avatar Brad Covey
Browse files

Added dynamic column creator for diagnostic columns, added csv generator

parent ab56f05f

diagnostic_csv.py

0 → 100644
+62 −0
Original line number Diff line number Diff line 
import pandas as pd

from sqlalchemy import func

from sqlalchemy.orm import defer, Load



import argparse



import models





def main():

	session = models.get_session()

	missions = session.query(models.missions).all()

	for m in missions:

		models.refresh()

		q = session.query(

				models.GliderLogs,

				models.Surfacings,

				models.Telemetry,

				models.EWO,

				models.Diagnostic

			).join(

				models.Diagnostic

			).join(

				models.Surfacings

			).join(

				models.Telemetry

			).join(

				models.EWO

			).filter(

				models.GliderLogs.missionID == m.id

			)

		q = q.options(

			Load(models.GliderLogs).defer('id'),

			Load(models.Surfacings).defer('id'),

			Load(models.Surfacings).defer('missionID'),

			Load(models.Telemetry).defer('id'),

			Load(models.Telemetry).defer('missionID'),

			Load(models.Telemetry).defer('gliderLogID'),

			Load(models.Telemetry).defer('datetime'),

			Load(models.EWO).defer('id'),

			Load(models.EWO).defer('missionID'),

			Load(models.EWO).defer('gliderLogID'),

			Load(models.EWO).defer('datetime'),

			Load(models.Diagnostic).defer('id'),

			Load(models.Diagnostic).defer('missionID'),

			Load(models.Diagnostic).defer('gliderLogID'),

			Load(models.Diagnostic).defer('datetime'),

		)

		df = pd.read_sql(q.statement, q.session.bind)

		glider = session.query(models.gliders).filter(

			models.gliders.columns.id == m.gliderID

		).one()

		out_filename = 'm%s_%s_%s.csv' % (

			                m.id,

			                glider.name,

			                str(m.startTime).replace(':', '_').replace(' ', '-')

		                )

		df.to_csv(out_filename, index=False)

	return



if __name__ == '__main__':

	main()

models.py

+47 −1
Original line number Diff line number Diff line 
"""A set of database models that will be used to store live slocum data"""

import os



import migrate.changeset

from sqlalchemy.ext.automap import automap_base

from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Float, DateTime, ForeignKeyConstraint, ForeignKey, Enum, UniqueConstraint, Boolean

from sqlalchemy.orm import sessionmaker





import atexit



import config
@@ -120,7 +122,7 @@ ewo = Table('ewo', m, 




m.create_all(engine)

m.reflect(engine)

m.reflect(bind=engine)

Base = automap_base(metadata=m)

Base.prepare()

Session = sessionmaker(bind=engine)
@@ -134,6 +136,50 @@ Telemetry = Base.classes.telemetry 
Aborts = Base.classes.aborts

EWO = Base.classes.ewo



def refresh():

    global m

    global Base

    global Session

    global engine



    global Gliders

    global Missions

    global Diagnostic

    global Surfacings

    global GliderLogs

    global Telemetry

    global Aborts

    global EWO



    m.reflect(bind=engine)

    Base = automap_base(metadata=m)

    Base.prepare(engine, reflect=True)

    Session = sessionmaker(bind=engine)



    Gliders = Base.classes.gliders

    Missions = Base.classes.missions

    Diagnostic = Base.classes.diagnostic

    Surfacings = Base.classes.surfacings

    GliderLogs = Base.classes.gliderLogs

    Telemetry = Base.classes.telemetry

    Aborts = Base.classes.aborts

    EWO = Base.classes.ewo





def update_diagnostic_columns(surface_message, already_defined):

    m.bind = engine

    for s in surface_message.sensors:

        if s not in already_defined and not hasattr(Diagnostic, s):

            print "Creating new diagnostic column: %s" % s

            try:

                float(surface_message.sensors[s].value)

                col = Column(s, Float)

            except ValueError as e:

                col = Column(s, String)

            col.create(Diagnostic.__table__)

    refresh()

    return





def get_session():

    return Session()

parse_surfacelogs.py

+10 −1
Original line number Diff line number Diff line
@@ -217,8 +217,11 @@ def store_surfacelog(surface_log): 
        mWaterVY=m_water_vy

    )

    session.add(telemetry_row)

    session.commit()



    try:

        already_defined = ['m_battery', 'm_leakdetect_voltage', 'm_vacuum', 'm_coulomb_amphr', 'm_coulomb_amphr_total', 'm_leakdetect_voltage_forward', 'm_water_vx', 'm_water_vy']

        models.update_diagnostic_columns(surface_message, already_defined)

        diagnostic_row = models.Diagnostic(

            missionID=mission_id,

            gliderLogID=glider_log.id,
@@ -227,9 +230,15 @@ def store_surfacelog(surface_log): 
            leakdetectVoltage=float(surface_message.sensors['m_leakdetect_voltage'].value),

            vacuum=float(surface_message.sensors['m_vacuum'].value),

            coulombAmphr=float(surface_message.sensors['m_coulomb_amphr'].value),

            coulombAmphrTotal=float(surface_message.sensors['m_battery'].value),

            coulombAmphrTotal=float(surface_message.sensors['m_coulomb_amphr_total'].value),

            leakdetectVoltageForward=float(surface_message.sensors['m_leakdetect_voltage_forward'].value)

        )

        for s in surface_message.sensors:

            if s not in already_defined and hasattr(diagnostic_row, s):

                val = surface_message.sensors[s].value

                if 'lat' in s or 'lon' in s:

                    val = dinkum.webb_to_decdeg(float(val))

                setattr(diagnostic_row, s, val)

        session.add(diagnostic_row)

    except KeyError:

        print "Incomplete logfile: %s Skipping." % surface_log.log_filename