Commit 4a1fe382 authored by Brad Covey's avatar Brad Covey

Added dynamic column creator for diagnostic columns, added csv generator

parent ab56f05f
import pandas as pd
from sqlalchemy import func
from sqlalchemy.orm import defer, Load
import argparse
import models
def main():
session = models.get_session()
missions = session.query(models.missions).all()
for m in missions:
models.refresh()
q = session.query(
models.GliderLogs,
models.Surfacings,
models.Telemetry,
models.EWO,
models.Diagnostic
).join(
models.Diagnostic
).join(
models.Surfacings
).join(
models.Telemetry
).join(
models.EWO
).filter(
models.GliderLogs.missionID == m.id
)
q = q.options(
Load(models.GliderLogs).defer('id'),
Load(models.Surfacings).defer('id'),
Load(models.Surfacings).defer('missionID'),
Load(models.Telemetry).defer('id'),
Load(models.Telemetry).defer('missionID'),
Load(models.Telemetry).defer('gliderLogID'),
Load(models.Telemetry).defer('datetime'),
Load(models.EWO).defer('id'),
Load(models.EWO).defer('missionID'),
Load(models.EWO).defer('gliderLogID'),
Load(models.EWO).defer('datetime'),
Load(models.Diagnostic).defer('id'),
Load(models.Diagnostic).defer('missionID'),
Load(models.Diagnostic).defer('gliderLogID'),
Load(models.Diagnostic).defer('datetime'),
)
df = pd.read_sql(q.statement, q.session.bind)
glider = session.query(models.gliders).filter(
models.gliders.columns.id == m.gliderID
).one()
out_filename = 'm%s_%s_%s.csv' % (
m.id,
glider.name,
str(m.startTime).replace(':', '_').replace(' ', '-')
)
df.to_csv(out_filename, index=False)
return
if __name__ == '__main__':
main()
"""A set of database models that will be used to store live slocum data"""
import os
import migrate.changeset
from sqlalchemy.ext.automap import automap_base
from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Float, DateTime, ForeignKeyConstraint, ForeignKey, Enum, UniqueConstraint, Boolean
from sqlalchemy.orm import sessionmaker
import atexit
import config
......@@ -120,7 +122,7 @@ ewo = Table('ewo', m,
m.create_all(engine)
m.reflect(engine)
m.reflect(bind=engine)
Base = automap_base(metadata=m)
Base.prepare()
Session = sessionmaker(bind=engine)
......@@ -134,6 +136,50 @@ Telemetry = Base.classes.telemetry
Aborts = Base.classes.aborts
EWO = Base.classes.ewo
def refresh():
global m
global Base
global Session
global engine
global Gliders
global Missions
global Diagnostic
global Surfacings
global GliderLogs
global Telemetry
global Aborts
global EWO
m.reflect(bind=engine)
Base = automap_base(metadata=m)
Base.prepare(engine, reflect=True)
Session = sessionmaker(bind=engine)
Gliders = Base.classes.gliders
Missions = Base.classes.missions
Diagnostic = Base.classes.diagnostic
Surfacings = Base.classes.surfacings
GliderLogs = Base.classes.gliderLogs
Telemetry = Base.classes.telemetry
Aborts = Base.classes.aborts
EWO = Base.classes.ewo
def update_diagnostic_columns(surface_message, already_defined):
m.bind = engine
for s in surface_message.sensors:
if s not in already_defined and not hasattr(Diagnostic, s):
print "Creating new diagnostic column: %s" % s
try:
float(surface_message.sensors[s].value)
col = Column(s, Float)
except ValueError as e:
col = Column(s, String)
col.create(Diagnostic.__table__)
refresh()
return
def get_session():
return Session()
......
......@@ -217,8 +217,11 @@ def store_surfacelog(surface_log):
mWaterVY=m_water_vy
)
session.add(telemetry_row)
session.commit()
try:
already_defined = ['m_battery', 'm_leakdetect_voltage', 'm_vacuum', 'm_coulomb_amphr', 'm_coulomb_amphr_total', 'm_leakdetect_voltage_forward', 'm_water_vx', 'm_water_vy']
models.update_diagnostic_columns(surface_message, already_defined)
diagnostic_row = models.Diagnostic(
missionID=mission_id,
gliderLogID=glider_log.id,
......@@ -227,9 +230,15 @@ def store_surfacelog(surface_log):
leakdetectVoltage=float(surface_message.sensors['m_leakdetect_voltage'].value),
vacuum=float(surface_message.sensors['m_vacuum'].value),
coulombAmphr=float(surface_message.sensors['m_coulomb_amphr'].value),
coulombAmphrTotal=float(surface_message.sensors['m_battery'].value),
coulombAmphrTotal=float(surface_message.sensors['m_coulomb_amphr_total'].value),
leakdetectVoltageForward=float(surface_message.sensors['m_leakdetect_voltage_forward'].value)
)
for s in surface_message.sensors:
if s not in already_defined and hasattr(diagnostic_row, s):
val = surface_message.sensors[s].value
if 'lat' in s or 'lon' in s:
val = dinkum.webb_to_decdeg(float(val))
setattr(diagnostic_row, s, val)
session.add(diagnostic_row)
except KeyError:
print "Incomplete logfile: %s Skipping." % surface_log.log_filename
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment