Commit ef01c0c7 authored by Chris Holbrook's avatar Chris Holbrook

Merge branch 'hotfix-data-loading' into 'master'

Hotfix data loading

Closes #52

See merge request !8
parents d626f119 206d3d56
......@@ -2,8 +2,8 @@ Package: glatos
Type: Package
Title: A package for the Great Lakes Acoustic Telemetry Observation System
Description: Functions useful to members of the Great Lakes Acoustic Telemetry Observation System www.glatos.glos.us; many more broadly relevant to simulating, processing, analysing, and visualizing acoustic telemetry data.
Version: 0.2.5
Date: 2018-03-14
Version: 0.2.6
Date: 2018-06-11
Depends: R (>= 3.2.0)
Imports:
cellranger,
......@@ -28,8 +28,8 @@ Authors@R: c(
role = c("cre", "aut")),
person("Todd", "Hayden", role = "aut"),
person("Thomas", "Binder", role = "aut"),
person("Jon", "Pye", role = "ctb"),
person("Alex", "Nunes", role = "ctb"),
person("Jon", "Pye", role = "aut"),
person("Alex", "Nunes", role = "aut"),
person("Angela", "Dini", role = "ctb"))
License: GPL-2
LazyLoad: yes
......
# glatos 0.2.6
#### 2018-06-11
### Minor changes and bug fixes
- read_otn_detections
- fix bug that indexed columns by position, meaning columns had to be
orderd exactly as in spec (issue #52)
- read_glatos_detections
- fix bug that indexed columns by position, meaning columns had to be
orderd exactly as in spec (issue #52)
- read_glatos_receivers
- fix bug that indexed columns by position, meaning columns had to be
orderd exactly as in spec (issue #52)
----
# glatos 0.2.5
#### 2018-03-14
......
......@@ -52,7 +52,7 @@ read_glatos_detections <- function(det_file, version = NULL) {
#Identify detection file version
id_det_version <- function(det_file){
det_col_names <- names(data.table::fread(det_file, nrows = 0))
if(all(glatos:::glatos_detection_schema$v1.3$name == det_col_names)) {
if(all(glatos:::glatos_detection_schema$v1.3$name %in% det_col_names)) {
return("1.3")
} else {
stop("Detection file version could not be identified.")
......@@ -63,7 +63,7 @@ read_glatos_detections <- function(det_file, version = NULL) {
version <- id_det_version(det_file)
} else if (!(paste0("v",version) %in%
names(glatos:::glatos_detection_schema))) {
stop(paste0("Detection file version ",version," is not supported."))
stop(paste0("Detection file version ", version," is not supported."))
}
#-Detections v1.3----------------------------------------------------------------
......@@ -80,12 +80,19 @@ read_glatos_detections <- function(det_file, version = NULL) {
#coerce timestamps to POSIXct; note that with fastPOSIXct raw
# timestamp must be in UTC; and tz argument sets the tzone attr only
for (j in timestamp_cols) data.table::set(dtc, j = j,
value = fasttime::fastPOSIXct(dtc[[j]], tz = "UTC"))
for (j in timestamp_cols) data.table::set(dtc,
j = glatos_detection_schema[["v1.3"]]$name[j],
value = fasttime::fastPOSIXct(
dtc[[glatos_detection_schema[["v1.3"]]$name[j]]],
tz = "UTC"))
#coerce dates to date
for (j in date_cols) {
data.table::set(dtc, j = j, value = ifelse(dtc[[j]] == "", NA, dtc[[j]]))
data.table::set(dtc, j = j, value = as.Date(dtc[[j]]))
data.table::set(dtc, j = glatos_detection_schema[["v1.3"]]$name[j],
value = ifelse(dtc[[glatos_detection_schema[["v1.3"]]$name[j]]] == "",
NA,
dtc[[glatos_detection_schema[["v1.3"]]$name[j]]]))
data.table::set(dtc, j = glatos_detection_schema[["v1.3"]]$name[j],
value = as.Date(dtc[[glatos_detection_schema[["v1.3"]]$name[j]]]))
}
}
#-end v1.3----------------------------------------------------------------
......
......@@ -45,7 +45,7 @@ read_glatos_receivers <- function(rec_file, version = NULL) {
#Identify file version
id_file_version <- function(rec_file){
col_names <- names(data.table::fread(rec_file, nrows = 0))
if(all(glatos:::glatos_receivers_schema$v1.0$name == col_names)) {
if(all(glatos:::glatos_receivers_schema$v1.0$name %in% col_names)) {
return("1.0")
} else {
stop("Receiver location file version could not be identified.")
......@@ -73,12 +73,18 @@ read_glatos_receivers <- function(rec_file, version = NULL) {
#coerce timestamps to POSIXct; note that with fastPOSIXct raw
# timestamp must be in UTC; and tz argument sets the tzone attr only
for (j in timestamp_cols) data.table::set(rec, j = j,
value = fasttime::fastPOSIXct(rec[[j]], tz = "UTC"))
for (j in timestamp_cols) data.table::set(rec,
j = glatos_receivers_schema[["v1.0"]]$name[j],
value = fasttime::fastPOSIXct(
rec[[glatos_receivers_schema[["v1.0"]]$name[j]]],
tz = "UTC"))
#coerce dates to date
for (j in date_cols) {
data.table::set(rec, j = j, value = ifelse(rec[[j]] == "", NA, rec[[j]]))
data.table::set(rec, j = j, value = as.Date(rec[[j]]))
data.table::set(rec, j = glatos_receivers_schema[["v1.0"]]$name[j],
value = ifelse(rec[[glatos_receivers_schema[["v1.0"]]$name[j]]] == "",
NA, rec[[glatos_receivers_schema[["v1.0"]]$name[j]]]))
data.table::set(rec, j = glatos_receivers_schema[["v1.0"]]$name[j],
value = as.Date(rec[[glatos_receivers_schema[["v1.0"]]$name[j]]]))
}
}
#-end v1.0----------------------------------------------------------------
......
#' Read data from a OTN detection file
#'
#'
#' Read data from a standard OTN detection (csv) file and return
#' a data.frame of class \code{glatos_detections}.
#'
#' @param det_file A character string with path and name of detection file in
#' OTN detection extract format (*.csv). If only file name is given, then the
#' @param det_file A character string with path and name of detection file in
#' OTN detection extract format (*.csv). If only file name is given, then the
#' file must be located in the working directory.
#'
#' @details
#'
#' @details
#' Data are loaded using \code{\link[data.table]{fread}} package and timestamps
#' are coerced to POSIXct using the \code{\link[fasttime]{fastPOSIXct}}. All
#' times must be in UTC timezone per GLATOS standard.
#'
#' @details
#' Column names are changed to match GLATOS standard columns when possible.
#'
#' @details
#' Column names are changed to match GLATOS standard columns when possible.
#' Otherwise, OTN columns and column names are retained.
#'
#' @return A data.frame of class \code{glatos_detections} that includes OTN
#' columns that do not map directly to GLATOS columns.
#'
#' @return A data.frame of class \code{glatos_detections} that includes OTN
#' columns that do not map directly to GLATOS columns.
#'
#' @author A. Nunes, \email{anunes@dal.ca}
#'
......@@ -29,23 +29,25 @@
#'
#' @export
read_otn_detections <- function(det_file) {
# Create a named vector for the column classes
col_classes <- otn_detection_schema$type
names(col_classes) <- otn_detection_schema$name
timestamp_cols <- which(col_classes == "POSIXct")
date_cols <- which(col_classes == "Date")
col_classes[c(timestamp_cols, date_cols)] <- "character"
#read data
dtc <- data.table::fread(det_file, sep = ",", colClasses = col_classes,
na.strings = c("", "NA"))
na.strings = c("", "NA"))
#coerce timestamps to POSIXct; note that with fastPOSIXct raw
# timestamp must be in UTC; and tz argument sets the tzone attr only
for (j in timestamp_cols) data.table::set(dtc, j = j,
value = fasttime::fastPOSIXct(dtc[[j]], tz = "UTC"))
for (j in timestamp_cols) data.table::set(dtc, j = otn_detection_schema$name[j],
value = fasttime::fastPOSIXct(dtc[[otn_detection_schema$name[j]]], tz = "UTC"))
#coerce dates to date
for (j in date_cols) {
data.table::set(dtc, j = j, value = ifelse(dtc[[j]] == "", NA, dtc[[j]]))
data.table::set(dtc, j = j, value = as.Date(dtc[[j]]))
data.table::set(dtc, j = otn_detection_schema$name[j], value = ifelse(dtc[[otn_detection_schema$name[j]]] == "", NA, dtc[[otn_detection_schema$name[j]]]))
data.table::set(dtc, j = otn_detection_schema$name[j], value = as.Date(dtc[[otn_detection_schema$name[j]]]))
}
data.table::setnames(dtc, old=otn_detection_schema$name, new=otn_detection_schema$mapping)
dtc <- glatos_detections(dtc)
......
No preview for this file type
......@@ -19,15 +19,15 @@ detectedby character detectedby
receiver_group character receiver_sn
station character glatos_array
receiver character receiver
bottom_depth integer bottom_depth
receiver_depth integer receiver_depth
bottom_depth numeric bottom_depth
receiver_depth numeric receiver_depth
tagname character transmitter_id
codespace character transmitter_codespace
sensorname logical sensorname
sensorraw logical sensorraw
sensorname character sensorname
sensorraw numeric sensorraw
sensortype character sensortype
sensorvalue logical sensorvalue
sensorunit logical sensorunit
sensorvalue numeric sensorvalue
sensorunit character sensorunit
datecollected POSIXct detection_timestamp_utc
timezone character timezone
longitude numeric deploy_long
......@@ -38,8 +38,8 @@ monthcollected integer monthcollected
daycollected integer daycollected
julianday integer julianday
timeofday numeric timeofday
datereleasedtagger logical datereleasedtagger
datereleasedpublic logical datereleasedpublic
datereleasedtagger Date datereleasedtagger
datereleasedpublic Date datereleasedpublic
local_area character local_area
notes character notes
citation character citation
......
......@@ -7,12 +7,12 @@
read_otn_detections(det_file)
}
\arguments{
\item{det_file}{A character string with path and name of detection file in
OTN detection extract format (*.csv). If only file name is given, then the
\item{det_file}{A character string with path and name of detection file in
OTN detection extract format (*.csv). If only file name is given, then the
file must be located in the working directory.}
}
\value{
A data.frame of class \code{glatos_detections} that includes OTN
A data.frame of class \code{glatos_detections} that includes OTN
columns that do not map directly to GLATOS columns.
}
\description{
......@@ -24,7 +24,7 @@ Data are loaded using \code{\link[data.table]{fread}} package and timestamps
are coerced to POSIXct using the \code{\link[fasttime]{fastPOSIXct}}. All
times must be in UTC timezone per GLATOS standard.
Column names are changed to match GLATOS standard columns when possible.
Column names are changed to match GLATOS standard columns when possible.
Otherwise, OTN columns and column names are retained.
}
\examples{
......
......@@ -26,3 +26,21 @@ test_that("lamprey_detections gives expected result", {
# Check if expected and actual results are the same
expect_equal(ld, lamprey_detections)
})
#test for mixed up column order
lamprey_detections2 <- ld[, c(2, 1, 3:27, 29, 28, 30)]
#write data frame with mixed up columns to temp file
temp_file <- tempfile()
write.csv(lamprey_detections2, temp_file, row.names = FALSE)
ld2 <- read_glatos_detections(temp_file)
test_that("lamprey_detections with mixed columns gives expected result", {
# Check if expected and actual results are the same
expect_equal(ld2, lamprey_detections2)
})
......@@ -13,3 +13,19 @@ test_that("read_glatos_receivers gives expected result", {
# Check if expected and actual results are the same
expect_equal(rec, sample_receivers)
})
#test for mixed up column order
sample_receivers2 <- rec[, c(1:8, 10, 9, 11:23)]
#write data frame with mixed up columns to temp file
temp_file <- tempfile()
write.csv(sample_receivers2, temp_file, row.names = FALSE)
rec2 <- read_glatos_receivers(temp_file)
test_that("read_glatos_receivers with mixed columns gives expected result", {
# Check if expected and actual results are the same
expect_equal(rec2, sample_receivers2)
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment