Commit f738b002 authored by Brad Covey's avatar Brad Covey

Added docstrings and more inline documentation

parent 7ef51047
A collection of functions for parsing a directory full of raw imagenex 853 files
from datetime import timedelta
import glob
import os.path
......@@ -15,6 +17,24 @@ debug = True
def process_directory(raw_file_dir, data_filename, header_filename):
Parse a directory of raw 853 files and output the result as CSV files.
Outputs header metadata and data into separate files
raw_file_dir : String
The path to a directory containing raw .853 files
data_filename : String
The path to the csv file where data will be stored
header_filename : String
The path to the csv file where header metadata will be stored
List of files that have been processed
processedListFile = open(os.path.join(os.path.split(os.path.realpath(__file__))[0], "853toCSV.pkl"), "r")
processedFiles = pickle.load(processedListFile)
......@@ -78,7 +98,8 @@ def process_directory(raw_file_dir, data_filename, header_filename):
return processedFiles
def process_mission(mission_id):
def process_mission(mission_id, outputDir):
"""Automate processing for a glider mission. Specific to Dal Glider Group"""
cursor, conn = get_database_connection()
# Repeat the parsing process for each glider
for gliderName in ['otn200', 'otn201']:
"""Configuration for database access. Only necessary if database interaction is desired"""
user = None
password = None
hostname = None
......@@ -8,6 +8,7 @@ import db_config
def date_or_orig(obj):
"""Attempt to convert obj to a date, return obj if unsuccessful"""
return date(*map(int, obj.split('-', 2)))
except ValueError:
......@@ -15,6 +16,7 @@ def date_or_orig(obj):
def datetime_or_orig(obj): # cribbed from MySQLdb2.0 revisions, updated to handle fractional seconds.
"""Attempt to convert obj to a datetime, return obj if unsucessful"""
if ' ' in obj:
sep = ' '
elif 'T' in obj:
......@@ -60,6 +62,7 @@ def timestamp_or_orig(s):
def get_database_connection(host=db_config.hostname, user=db_config.user,
db=db_config.database, passwd=db_config.password,
port=db_config.port, cursorclass='MySQLdb.cursors.DictCursor'):
"""Return a database cursor and connection using for access parameters"""
conv_dict = MySQLdb.converters.conversions
conv_dict[12] = datetime_or_orig # Override DateTime_or_None function. Works.
......@@ -8,6 +8,7 @@ from datetime import datetime
class Message:
"""A file-like object that can be used to create virtual files"""
def __init__(self, filename):
self.byteStream = []
self.position = 0
......@@ -40,6 +41,22 @@ class Message:
# The first element in the tuple is the header represented as a dictionary
# The second element is an array of peak-peak voltage data
def parse853Message(filename):
Parse a message out of an 853 file
Checks if the message is full resolution or reduced and calls the relevant parser
filename : file | Message
The file handle-like object of an 853 file to be parsed
The header data parsed out of the 853 file
The data parsed out of the 853 file
infile = filename
......@@ -53,6 +70,7 @@ def parse853Message(filename):
print "Can't identify message"
def parse256BytePacket(infile):
"""Parse a full resolution 853 file"""
# The variables that will store the data which gets parsed out of the binary file
header = {}
data = []
......@@ -154,7 +172,6 @@ def parse256BytePacket(infile):
# Accepts a string containing the name of the 18Byte file and a datetime.datetime object representing the creation date for the file
# Returns a unix timestamp
def get18Bytetimestamp(filename, fileModifiedDate):
day = int(filename[2:4])
hour = int(filename[4:6])
minute = int(filename[6:8])
......@@ -166,6 +183,7 @@ def get18Bytetimestamp(filename, fileModifiedDate):
return time.mktime(datetime(year,month,day,hour,minute).timetuple())
def parse18BytePacket(infile):
"""Parse a reduced 853 file"""
# The variables that will store the data which gets parsed out of the binary file
header = {}
data = []
......@@ -206,7 +224,25 @@ def parse18BytePacket(infile):
return (header, data)
def parse853File(infile, returnHeaderAsDict=True):
Split an 853 file into individual messages and parse them.
infile : file
The filehandle for the 853 file to be parsed
returnHeaderAsDict : bool
Whether the header should be returned as a dict or a list
dict | list
The headers parsed out of the data file
A 2-dimensional matrix of peak-peak voltages parsed out of the 853 file
messages = []
termByte = int('0xFC', 16)
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment