From 65ffa196b6ddb8f4f2f17a4f825710db5b0ebb9b Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 26 Dec 2021 09:04:57 -0700 Subject: [PATCH 01/15] Update getFinancials.R --- R/getFinancials.R | 86 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index a9fe3699..9fd5157e 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,10 +1,34 @@ `getFinancials` <- -getFin <- function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { - src <- match.arg(src, "google") - if (src != "google") { - stop("src = ", sQuote(src), " is not implemented") +getFin <- + function(Symbols, env=parent.frame(), src = "tiingo", auto.assign=TRUE, from = Sys.Date()-720, to=Sys.Date(), api.key=NULL, ...) { + importDefaults("getFinancials") + #TODO: add documentation and tests + + src <- match.arg(src, "tiingo") + if (src != "tiingo") stop("src = ", sQuote(src), " is not implemented") + + if(is.null(env)) + auto.assign <- FALSE + if(!auto.assign && length(Symbols)>1) + stop("must use auto.assign=TRUE for multiple Symbols requests") + + Symbols <- strsplit(Symbols, ";") + ret.sym <- list() + for(s in Symbols) { + z <- try(structure(do.call(paste("getFinancials", src, sep = "."), + args = list(Symbol = s, from = from, to = to, api.key = api.key, ...)), + symbol = s, class = "financials", src = src, updated = Sys.time())) + if (auto.assign) { + if (inherits(z, "financials")) { + new.sym <- paste(gsub(":", ".", Symbol.name), "f", sep = ".") + assign(new.sym, z, env) + ret.sym[[length(ret.sym + 1)]] <- new.sym + } + } else { + return(z) + } } - getFinancials.google(Symbol, env, auto.assign = auto.assign, ...) + return(unlist(ret.sym)) } getFinancials.google <- @@ -25,7 +49,7 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { `viewFin` <- `viewFinancials` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), - subset = NULL) { + subset = "") { if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) type <- match.arg(toupper(type[1]),c('BS','IS','CF')) period <- match.arg(toupper(period[1]),c('A','Q')) @@ -37,19 +61,41 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { A='Annual', Q='Quarterly') - if(is.null(subset)) { - message(paste(statements[[period]],statements[[type]],'for',attr(x,'symbol'))) - return(x[[type]][[period]]) - } else { - tmp.table <- as.matrix(as.xts(t(x[[type]][[period]]),dateFormat='Date')[subset]) - dn1 <- rownames(tmp.table) - dn2 <- colnames(tmp.table) - tmp.table <- t(tmp.table)[, NROW(tmp.table):1] - if(is.null(dim(tmp.table))) { - dim(tmp.table) <- c(NROW(tmp.table),1) - dimnames(tmp.table) <- list(dn2,dn1) + message(paste(statements[[period]],statements[[type]],'for',attr(x,'symbol'))) + return(t(x[[type]][[period]][subset])) +} + +getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { + URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?startDate=%s&endDate=%s&token=%s", Symbol, from, to, api.key) + d <- jsonlite::fromJSON(URL) + + r <- list(periods = data.frame( + type = ifelse(d$quarter == 0, "A", "Q"), + year = d$year, + quarter = ifelse(d$quarter == 0, NA_integer_, d$quarter), + ending = as.Date(d$date) + ) + ) + + #tiiingo section names + name.map <- list(balanceSheet = "BS", + incomeStatement = "IS", + cashFlow ="CF") + + #merge into a single df with columns for each period + for (st in names(name.map)) { + nm <- name.map[[st]] + if (!is.null(nm)) { + # suppress duplicate column names on merge + mdf <- suppressWarnings(Reduce(function(x,y){merge(x, y, all = TRUE, by = "dataCode")}, d$statementData[[st]])) + m <- sapply(mdf[,-1], as.numeric) #convert merged dataframe to numeric matrix and transpose + rownames(m) <- mdf[[1]] + m <- t(m) #transpose for conversion to xts + q.idx <- which(r$periods$type == "Q") + r[[nm]] <- list(Q = xts(m[q.idx,, drop = FALSE], order.by = r$periods$ending[q.idx]), + A = xts(m[-q.idx,, drop = FALSE], order.by = r$periods$ending[-q.idx]) + ) } - message(paste(statements[[period]],statements[[type]],'for',attr(x,'symbol'))) - return(tmp.table) } -} + return(r) +} \ No newline at end of file From d864c3e9d1089f7e3e7323c9879a3bffab02d359 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 26 Dec 2021 09:26:13 -0700 Subject: [PATCH 02/15] Update getFinancials.R --- R/getFinancials.R | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 9fd5157e..1ab07167 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -49,11 +49,10 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { `viewFin` <- `viewFinancials` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), - subset = "") { + subset = NULL) { if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) type <- match.arg(toupper(type[1]),c('BS','IS','CF')) - period <- match.arg(toupper(period[1]),c('A','Q')) - + period <- match.arg(toupper(period[1]),c('A','Q')) statements <- list(BS='Balance Sheet', IS='Income Statement', @@ -62,7 +61,11 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { Q='Quarterly') message(paste(statements[[period]],statements[[type]],'for',attr(x,'symbol'))) - return(t(x[[type]][[period]][subset])) + r <- x[[type]][[period]] + if (is.null(subset)) + return(r) + else + return(t(as.xts(t(r))[subset])) } getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { @@ -88,14 +91,12 @@ getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { if (!is.null(nm)) { # suppress duplicate column names on merge mdf <- suppressWarnings(Reduce(function(x,y){merge(x, y, all = TRUE, by = "dataCode")}, d$statementData[[st]])) - m <- sapply(mdf[,-1], as.numeric) #convert merged dataframe to numeric matrix and transpose + m <- sapply(mdf[,-1], as.numeric) #convert merged dataframe to numeric matrix rownames(m) <- mdf[[1]] - m <- t(m) #transpose for conversion to xts + colnames(m) <- as.character(r$periods$ending) q.idx <- which(r$periods$type == "Q") - r[[nm]] <- list(Q = xts(m[q.idx,, drop = FALSE], order.by = r$periods$ending[q.idx]), - A = xts(m[-q.idx,, drop = FALSE], order.by = r$periods$ending[-q.idx]) - ) + r[[nm]] <- list(Q = m[,q.idx, drop = FALSE],A = m[,-q.idx, drop = FALSE]) } } return(r) -} \ No newline at end of file +} From 71afb9e8c69c0d5a092567c15248f65588e05215 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 26 Dec 2021 10:00:28 -0700 Subject: [PATCH 03/15] updated man page. bug fixes for auto.assign path --- R/getFinancials.R | 15 ++++++++------- man/getFinancials.Rd | 19 ++++++++++++------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 1ab07167..13d7d755 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,9 +1,8 @@ `getFinancials` <- getFin <- - function(Symbols, env=parent.frame(), src = "tiingo", auto.assign=TRUE, from = Sys.Date()-720, to=Sys.Date(), api.key=NULL, ...) { + function(Symbols, env=parent.frame(), src = "tiingo", auto.assign=TRUE, from = Sys.Date()-720, to=Sys.Date(), ...) { importDefaults("getFinancials") #TODO: add documentation and tests - src <- match.arg(src, "tiingo") if (src != "tiingo") stop("src = ", sQuote(src), " is not implemented") @@ -14,15 +13,15 @@ getFin <- Symbols <- strsplit(Symbols, ";") ret.sym <- list() - for(s in Symbols) { + for(sym in Symbols) { z <- try(structure(do.call(paste("getFinancials", src, sep = "."), - args = list(Symbol = s, from = from, to = to, api.key = api.key, ...)), - symbol = s, class = "financials", src = src, updated = Sys.time())) + args = list(Symbol = sym, from = from, to = to, ...)), + symbol = sym, class = "financials", src = src, updated = Sys.time())) if (auto.assign) { if (inherits(z, "financials")) { - new.sym <- paste(gsub(":", ".", Symbol.name), "f", sep = ".") + new.sym <- paste(gsub(":", ".", sym), "f", sep = ".") assign(new.sym, z, env) - ret.sym[[length(ret.sym + 1)]] <- new.sym + ret.sym[[length(ret.sym) + 1]] <- new.sym } } else { return(z) @@ -69,8 +68,10 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { } getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { + importDefaults("getFinancials.tiingo") URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?startDate=%s&endDate=%s&token=%s", Symbol, from, to, api.key) d <- jsonlite::fromJSON(URL) + if (length(d) == 0) stop("No data returned for Symbol:", Symbol) r <- list(periods = data.frame( type = ifelse(d$quarter == 0, "A", "Q"), diff --git a/man/getFinancials.Rd b/man/getFinancials.Rd index c2e2f981..b131d915 100644 --- a/man/getFinancials.Rd +++ b/man/getFinancials.Rd @@ -8,7 +8,7 @@ Download Income Statement, Balance Sheet, and Cash Flow Statements. } \usage{ -getFinancials(Symbol, env = parent.frame(), src = "google", +getFinancials(Symbol, env = parent.frame(), src = "tiingo", auto.assign = TRUE, ...) @@ -45,22 +45,27 @@ which statements to view - (A) for annual and (Q) for quarterly. } \value{ - Six individual matrices organized in a list of class \sQuote{financials}: + Six individual matrices and a data.frame organized in a list of class \sQuote{financials}: + \item{ periods }{ a data.frame containing Fiscal year and quarter corresponding to each statement contained in the results. + The \code{ending} column in this data.frame matches the column name in the statement matrix corresponding to period. + } \item{ IS }{ a list containing (Q)uarterly and (A)nnual Income Statements } \item{ BS }{ a list containing (Q)uarterly and (A)nnual Balance Sheets } \item{ CF }{ a list containing (Q)uarterly and (A)nnual Cash Flow Statements } + + Only the data structure itself is normalized. Individual statement matrixes use field names from the underlying source. + Please refer to the source documentation for definitions and explanations. } -\author{ Jeffrey A. Ryan } +\author{ Jeffrey A. Ryan, Ethan Smith } \note{ -As with all free data, you may be getting exactly what you pay for. -Sometimes that may be absolutely nothing. +currently only implemented for tiingo, which requres a paid license. } \examples{ \dontrun{ -getFinancials('JAVA') # returns JAVA.f to "env" +getFinancials('BA') # returns JAVA.f to "env" getFin('AAPL') # returns AAPL.f to "env" -viewFin(JAVA.f, "IS", "Q") # Quarterly Income Statement +viewFin(BA.f, "IS", "Q") # Quarterly Income Statement viewFin(AAPL.f, "CF", "A") # Annual Cash Flows str(AAPL.f) From 662dc7f6c9b2b933351eee7402222d9e96b4a133 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 26 Dec 2021 10:21:07 -0700 Subject: [PATCH 04/15] added tests --- man/getFinancials.Rd | 5 +++-- tests/test_getFinancials.R | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 tests/test_getFinancials.R diff --git a/man/getFinancials.Rd b/man/getFinancials.Rd index b131d915..6e0323dd 100644 --- a/man/getFinancials.Rd +++ b/man/getFinancials.Rd @@ -47,7 +47,8 @@ for quarterly. \value{ Six individual matrices and a data.frame organized in a list of class \sQuote{financials}: \item{ periods }{ a data.frame containing Fiscal year and quarter corresponding to each statement contained in the results. - The \code{ending} column in this data.frame matches the column name in the statement matrix corresponding to period. + This acts as a manifest for the contents of the rest of the structure. + The \code{ending} column matches the column name in the statement matrix corresponding to period. } \item{ IS }{ a list containing (Q)uarterly and (A)nnual Income Statements } \item{ BS }{ a list containing (Q)uarterly and (A)nnual Balance Sheets } @@ -62,7 +63,7 @@ currently only implemented for tiingo, which requres a paid license. } \examples{ \dontrun{ -getFinancials('BA') # returns JAVA.f to "env" +getFinancials('BA') # returns BA.f to "env" getFin('AAPL') # returns AAPL.f to "env" viewFin(BA.f, "IS", "Q") # Quarterly Income Statement diff --git a/tests/test_getFinancials.R b/tests/test_getFinancials.R new file mode 100644 index 00000000..7aa101cb --- /dev/null +++ b/tests/test_getFinancials.R @@ -0,0 +1,17 @@ +library(quantmod) + +# Tests for getFinancials + +# Checks for tiingo +# tiingo allows access to dow30 symbols for testing/development +apikey <- Sys.getenv("QUANTMOD_TIINGO_API_KEY") +if (nzchar(apikey)) { + aapl <- getFinancials("AAPL", src = "tiingo", api.key = apikey, + auto.assign = FALSE) + stopifnot(inherits(aapl, "financials")) + + #test multisymbol path with bad symbols + retsym <- getFinancials("AAPL;BA;UNKNOWNSYMBOL", src = "tiingo", + api.key = apikey, auto.assign = TRUE) + stopifnot(retsym == c("AAPL.f", "BA.f")) +} From e41d6c43ab774b869cc86fcf2e675d200fcaab6e Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 16 Jan 2022 13:58:07 -0700 Subject: [PATCH 05/15] support irregular result shape --- R/getFinancials.R | 84 +++++++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 13d7d755..856ed66f 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,8 +1,10 @@ `getFinancials` <- -getFin <- + getFin <- function(Symbols, env=parent.frame(), src = "tiingo", auto.assign=TRUE, from = Sys.Date()-720, to=Sys.Date(), ...) { importDefaults("getFinancials") - #TODO: add documentation and tests + #As much desired generic functionality, erro handlign and recovery has been moved into the master function + #source specific fucntions should just fetch data for a single symbol and be as lightweight as possible + #TODO: add tests src <- match.arg(src, "tiingo") if (src != "tiingo") stop("src = ", sQuote(src), " is not implemented") @@ -11,22 +13,28 @@ getFin <- if(!auto.assign && length(Symbols)>1) stop("must use auto.assign=TRUE for multiple Symbols requests") - Symbols <- strsplit(Symbols, ";") + Symbols <- strsplit(Symbols, ";")[[1]] ret.sym <- list() + failed.sym <- list() for(sym in Symbols) { z <- try(structure(do.call(paste("getFinancials", src, sep = "."), args = list(Symbol = sym, from = from, to = to, ...)), - symbol = sym, class = "financials", src = src, updated = Sys.time())) + symbol = sym, class = "financials", src = src, updated = Sys.time())) if (auto.assign) { if (inherits(z, "financials")) { new.sym <- paste(gsub(":", ".", sym), "f", sep = ".") assign(new.sym, z, env) ret.sym[[length(ret.sym) + 1]] <- new.sym - } + } else { + failed.sym[[length(failed.sym) + 1]] <- sym + } } else { - return(z) + return(z) } } + if (length(failed.sym) > 0) { + warning("Failed getting financials for ", paste(unlist(failed.sym), collapse = ";")) + } return(unlist(ret.sym)) } @@ -67,37 +75,55 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { return(t(as.xts(t(r))[subset])) } -getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { +nancials.tiingo <- function(Symbol, from, to, api.key, ...) { + #API Documentation: https://api.tiingo.com/documentation/fundamentals importDefaults("getFinancials.tiingo") - URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?startDate=%s&endDate=%s&token=%s", Symbol, from, to, api.key) + #while the api supports CSV, json is a bit more effecient over the wire + URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?startDate=%s&endDate=%s&token=%s", + Symbol, from, to, api.key) d <- jsonlite::fromJSON(URL) if (length(d) == 0) stop("No data returned for Symbol:", Symbol) - r <- list(periods = data.frame( - type = ifelse(d$quarter == 0, "A", "Q"), - year = d$year, - quarter = ifelse(d$quarter == 0, NA_integer_, d$quarter), - ending = as.Date(d$date) - ) - ) + periods = data.frame( + period = ifelse(d$quarter == 0, "A", "Q"), + year = d$year, + quarter = ifelse(d$quarter == 0, NA_integer_, d$quarter), + ending = as.Date(d$date) + ) - #tiiingo section names - name.map <- list(balanceSheet = "BS", + #tiiingo type to normalized types + statement.types <- list(balanceSheet = "BS", incomeStatement = "IS", cashFlow ="CF") + period.names <- c("Q","A") - #merge into a single df with columns for each period - for (st in names(name.map)) { - nm <- name.map[[st]] - if (!is.null(nm)) { - # suppress duplicate column names on merge - mdf <- suppressWarnings(Reduce(function(x,y){merge(x, y, all = TRUE, by = "dataCode")}, d$statementData[[st]])) - m <- sapply(mdf[,-1], as.numeric) #convert merged dataframe to numeric matrix - rownames(m) <- mdf[[1]] - colnames(m) <- as.character(r$periods$ending) - q.idx <- which(r$periods$type == "Q") - r[[nm]] <- list(Q = m[,q.idx, drop = FALSE],A = m[,-q.idx, drop = FALSE]) + r <- lapply(names(statement.types), function(st) { + #warning, some periods may be missing statement types + statements <- d$statementData[[st]] + if (length(statements) < 1) { + warning("No", statement.types[[statement.type]], "data for", Symbol) + z <-vector(mode='list', length = length(period.names)) + } else { + missing <- sapply(statements, is.null) + z <- lapply(period.names, function(pn) { + selected <- which(periods$period == pn & !missing) + if (length(selected) > 0) { + merged <- Reduce(function(x,y) { + suppressWarnings(merge(x, y, all = TRUE, by = "dataCode")) #warnings on duplicate column names + }, statements[selected]) + #convert merged dataframe to numeric matrix, warnings on NA numeric conversion + m <- suppressWarnings(sapply(merged[,-1], as.double)) + if (is.null(dim(m))) m <- as.matrix(m, nrow = nrow(merged)) + rownames(m) <- merged[[1]] #names in first column + colnames(m) <- as.character(periods$ending[selected]) + } else m <- NULL + return(m) + }) } - } + names(z) <- period.names + return(z) + }) + names(r) <- statement.types + r$periods <- periods return(r) } From 4b24f6423237eb6643df2052bcad1b80a1559701 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 16 Jan 2022 15:52:49 -0700 Subject: [PATCH 06/15] minor fixes and cleanup --- R/getFinancials.R | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 856ed66f..93e21a53 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -55,8 +55,8 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { } `viewFin` <- -`viewFinancials` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), - subset = NULL) { + `viewFinancials` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), + subset = NULL) { if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) type <- match.arg(toupper(type[1]),c('BS','IS','CF')) period <- match.arg(toupper(period[1]),c('A','Q')) @@ -69,13 +69,13 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { message(paste(statements[[period]],statements[[type]],'for',attr(x,'symbol'))) r <- x[[type]][[period]] - if (is.null(subset)) + if (is.null(r) || is.null(subset)) return(r) else return(t(as.xts(t(r))[subset])) } -nancials.tiingo <- function(Symbol, from, to, api.key, ...) { +getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { #API Documentation: https://api.tiingo.com/documentation/fundamentals importDefaults("getFinancials.tiingo") #while the api supports CSV, json is a bit more effecient over the wire @@ -91,21 +91,22 @@ nancials.tiingo <- function(Symbol, from, to, api.key, ...) { ending = as.Date(d$date) ) - #tiiingo type to normalized types - statement.types <- list(balanceSheet = "BS", - incomeStatement = "IS", - cashFlow ="CF") - period.names <- c("Q","A") + #normalized to tiingo mappings + statement.types <- list(BS = "balanceSheet", + IS = "incomeStatement", + CF = "cashFlow") + statement.periods <- list(A='Annual', + Q='Quarterly') - r <- lapply(names(statement.types), function(st) { + r <- lapply(statement.types, function(st) { #warning, some periods may be missing statement types statements <- d$statementData[[st]] if (length(statements) < 1) { warning("No", statement.types[[statement.type]], "data for", Symbol) - z <-vector(mode='list', length = length(period.names)) + z <- vector(mode='list', length = length(period.names)) } else { missing <- sapply(statements, is.null) - z <- lapply(period.names, function(pn) { + z <- lapply(names(statement.periods), function(pn) { selected <- which(periods$period == pn & !missing) if (length(selected) > 0) { merged <- Reduce(function(x,y) { @@ -120,10 +121,10 @@ nancials.tiingo <- function(Symbol, from, to, api.key, ...) { return(m) }) } - names(z) <- period.names + names(z) <- names(statement.periods) return(z) }) - names(r) <- statement.types + names(r) <- names(statement.types) r$periods <- periods return(r) } From 066aeb1058d96d4056940c153c0db65234c63e84 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:13:18 -0700 Subject: [PATCH 07/15] Update getFinancials.R migrated to CSV input for simplified processing and added support for asReported parameter --- R/getFinancials.R | 70 +++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 93e21a53..d001622b 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -75,56 +75,36 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { return(t(as.xts(t(r))[subset])) } -getFinancials.tiingo <- function(Symbol, from, to, api.key, ...) { +getFinancials.tiingo <- function(Symbol, from, to, as.reported = FALSE, api.key, ...) { #API Documentation: https://api.tiingo.com/documentation/fundamentals importDefaults("getFinancials.tiingo") - #while the api supports CSV, json is a bit more effecient over the wire - URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?startDate=%s&endDate=%s&token=%s", - Symbol, from, to, api.key) - d <- jsonlite::fromJSON(URL) - if (length(d) == 0) stop("No data returned for Symbol:", Symbol) - - periods = data.frame( - period = ifelse(d$quarter == 0, "A", "Q"), - year = d$year, - quarter = ifelse(d$quarter == 0, NA_integer_, d$quarter), - ending = as.Date(d$date) - ) + URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?format=csv&startDate=%s&endDate=%s&asReported=%s&token=%s", + Symbol, from, to, tolower(as.reported), api.key) + d <- read.csv(URL) + if (ncol(d) == 1 && colnames(d) == "None") stop("No data returned for Symbol: ", Symbol) #normalized to tiingo mappings - statement.types <- list(BS = "balanceSheet", - IS = "incomeStatement", - CF = "cashFlow") - statement.periods <- list(A='Annual', - Q='Quarterly') + statement.types <- c(balanceSheet = "BS", + incomeStatement = "IS", + cashFlow = "CF") - r <- lapply(statement.types, function(st) { - #warning, some periods may be missing statement types - statements <- d$statementData[[st]] - if (length(statements) < 1) { - warning("No", statement.types[[statement.type]], "data for", Symbol) - z <- vector(mode='list', length = length(period.names)) - } else { - missing <- sapply(statements, is.null) - z <- lapply(names(statement.periods), function(pn) { - selected <- which(periods$period == pn & !missing) - if (length(selected) > 0) { - merged <- Reduce(function(x,y) { - suppressWarnings(merge(x, y, all = TRUE, by = "dataCode")) #warnings on duplicate column names - }, statements[selected]) - #convert merged dataframe to numeric matrix, warnings on NA numeric conversion - m <- suppressWarnings(sapply(merged[,-1], as.double)) - if (is.null(dim(m))) m <- as.matrix(m, nrow = nrow(merged)) - rownames(m) <- merged[[1]] #names in first column - colnames(m) <- as.character(periods$ending[selected]) - } else m <- NULL - return(m) - }) - } - names(z) <- names(statement.periods) - return(z) + d <- d[d$statementType %in% names(statement.types) & d$quarter %in% (0:4),] + d$period <- ifelse(d$quarter == 0, "A", "Q") + #partition by statement type + tsubs <- split(d[, c("date", "dataCode","value", "period")], statement.types[d$statementType]) + r <- lapply(tsubs, function(tsub) { + dsubs <- split(tsub, tsub$period) + #partition by period (Q or A), pivot and convert to a matrix + lapply(dsubs, function(dsub) { + pivot <- reshape(dsub[, c("date", "dataCode", "value")], + timevar = "date", idvar = "dataCode", direction = "wide") + rownames(pivot) <- pivot[[1]] + pivot <- pivot[, -1, drop = FALSE] + colnames(pivot) <- gsub("^value\\.", "", colnames(pivot)) + return(as.matrix(pivot)) + }) }) - names(r) <- names(statement.types) - r$periods <- periods + + r$periods <- unique(d[, c("date", "year", "quarter", "period")]) return(r) } From 6e4ea2158f3cdde398f4e1dbf67071374416e664 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Mon, 17 Jan 2022 13:49:18 -0700 Subject: [PATCH 08/15] Update getFinancials.R formatting cleanup --- R/getFinancials.R | 67 ++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 45 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index d001622b..1cd15e6d 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,62 +1,38 @@ -`getFinancials` <- - getFin <- - function(Symbols, env=parent.frame(), src = "tiingo", auto.assign=TRUE, from = Sys.Date()-720, to=Sys.Date(), ...) { - importDefaults("getFinancials") - #As much desired generic functionality, erro handlign and recovery has been moved into the master function +`getFinancials` <- getFin <- + function(Symbols, env=parent.frame(), src="tiingo", auto.assign=TRUE, from=Sys.Date()-720, to=Sys.Date(), ...) { + #As much desired generic functionality, error handlign and recovery has been moved into the master function #source specific fucntions should just fetch data for a single symbol and be as lightweight as possible - #TODO: add tests - src <- match.arg(src, "tiingo") - if (src != "tiingo") stop("src = ", sQuote(src), " is not implemented") + importDefaults("getFinancials") + call.name <- paste("getFinancials", src, sep = ".") + Symbols <- strsplit(Symbols, ";")[[1]] + if (length(find(call.name, mode = "function")) < 1) + stop("src = ", sQuote(src), " is not implemented") if(is.null(env)) auto.assign <- FALSE - if(!auto.assign && length(Symbols)>1) + if(!auto.assign && length(Symbols) > 1) stop("must use auto.assign=TRUE for multiple Symbols requests") - Symbols <- strsplit(Symbols, ";")[[1]] ret.sym <- list() - failed.sym <- list() - for(sym in Symbols) { - z <- try(structure(do.call(paste("getFinancials", src, sep = "."), - args = list(Symbol = sym, from = from, to = to, ...)), - symbol = sym, class = "financials", src = src, updated = Sys.time())) + for (sym in Symbols) { + args <- list(Symbol = sym, from = from, to = to, ...) + fin <- try(structure(do.call(call.name, args = args), + symbol = sym, class = "financials", src = src, updated = Sys.time())) if (auto.assign) { - if (inherits(z, "financials")) { + if (inherits(fin, "financials")) { new.sym <- paste(gsub(":", ".", sym), "f", sep = ".") - assign(new.sym, z, env) + assign(new.sym, fin, env) ret.sym[[length(ret.sym) + 1]] <- new.sym - } else { - failed.sym[[length(failed.sym) + 1]] <- sym } } else { - return(z) + return(fin) } } - if (length(failed.sym) > 0) { - warning("Failed getting financials for ", paste(unlist(failed.sym), collapse = ";")) - } return(unlist(ret.sym)) } -getFinancials.google <- -function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { - msg <- paste0(sQuote("getFinancials.google"), " is defunct.", - "\nGoogle Finance stopped providing data in March, 2018.", - "\nYou could try some of the data sources via Quandl instead.", - "\nSee help(\"Defunct\") and help(\"quantmod-defunct\")") - .Defunct("Quandl", "quantmod", msg = msg) -} - - -`print.financials` <- function(x, ...) { - cat('Financial Statement for',attr(x,'symbol'),'\n') - cat('Retrieved from',attr(x,'src'),'at',format(attr(x,'updated')),'\n') - cat('Use "viewFinancials" or "viewFin" to view\n') -} - -`viewFin` <- - `viewFinancials` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), - subset = NULL) { +`viewFin` <- `viewFinancials` <- + function(x, type=c('BS','IS','CF'), period=c('A','Q'), subset = NULL) { if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) type <- match.arg(toupper(type[1]),c('BS','IS','CF')) period <- match.arg(toupper(period[1]),c('A','Q')) @@ -75,7 +51,7 @@ function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { return(t(as.xts(t(r))[subset])) } -getFinancials.tiingo <- function(Symbol, from, to, as.reported = FALSE, api.key, ...) { +getFinancials.tiingo <- function(Symbol, from, to, as.reported=FALSE, api.key, ...) { #API Documentation: https://api.tiingo.com/documentation/fundamentals importDefaults("getFinancials.tiingo") URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?format=csv&startDate=%s&endDate=%s&asReported=%s&token=%s", @@ -85,8 +61,8 @@ getFinancials.tiingo <- function(Symbol, from, to, as.reported = FALSE, api.key, #normalized to tiingo mappings statement.types <- c(balanceSheet = "BS", - incomeStatement = "IS", - cashFlow = "CF") + incomeStatement = "IS", + cashFlow = "CF") d <- d[d$statementType %in% names(statement.types) & d$quarter %in% (0:4),] d$period <- ifelse(d$quarter == 0, "A", "Q") @@ -96,6 +72,7 @@ getFinancials.tiingo <- function(Symbol, from, to, as.reported = FALSE, api.key, dsubs <- split(tsub, tsub$period) #partition by period (Q or A), pivot and convert to a matrix lapply(dsubs, function(dsub) { + if (NROW(dsub) < 1) return(NULL) pivot <- reshape(dsub[, c("date", "dataCode", "value")], timevar = "date", idvar = "dataCode", direction = "wide") rownames(pivot) <- pivot[[1]] From eb19081de5973a2de7685f8ebd66e9c854c2e596 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Mon, 17 Jan 2022 15:56:01 -0700 Subject: [PATCH 09/15] Update getFinancials.R more cleanup and commenting --- R/getFinancials.R | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 1cd15e6d..efb6ba1b 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,7 +1,7 @@ `getFinancials` <- getFin <- function(Symbols, env=parent.frame(), src="tiingo", auto.assign=TRUE, from=Sys.Date()-720, to=Sys.Date(), ...) { - #As much desired generic functionality, error handlign and recovery has been moved into the master function - #source specific fucntions should just fetch data for a single symbol and be as lightweight as possible + #As much generic functionality and error handlign has been moved into the master function + #source specific fucnimplementations should just fetch data for a single symbol and be as lightweight as possible importDefaults("getFinancials") call.name <- paste("getFinancials", src, sep = ".") @@ -31,8 +31,9 @@ return(unlist(ret.sym)) } -`viewFin` <- `viewFinancials` <- +`viewFinancials` <- `viewFin` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), subset = NULL) { + importDefaults("viewFinancials") if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) type <- match.arg(toupper(type[1]),c('BS','IS','CF')) period <- match.arg(toupper(period[1]),c('A','Q')) @@ -56,18 +57,16 @@ getFinancials.tiingo <- function(Symbol, from, to, as.reported=FALSE, api.key, . importDefaults("getFinancials.tiingo") URL <- sprintf("https://api.tiingo.com/tiingo/fundamentals/%s/statements?format=csv&startDate=%s&endDate=%s&asReported=%s&token=%s", Symbol, from, to, tolower(as.reported), api.key) - d <- read.csv(URL) + d <- suppressWarnings(read.csv(URL)) if (ncol(d) == 1 && colnames(d) == "None") stop("No data returned for Symbol: ", Symbol) - #normalized to tiingo mappings - statement.types <- c(balanceSheet = "BS", - incomeStatement = "IS", - cashFlow = "CF") - - d <- d[d$statementType %in% names(statement.types) & d$quarter %in% (0:4),] + stypes <- c(balanceSheet = "BS", incomeStatement = "IS", cashFlow = "CF") + d <- d[d$statementType %in% names(stypes) & d$quarter %in% (0:4),] d$period <- ifelse(d$quarter == 0, "A", "Q") - #partition by statement type - tsubs <- split(d[, c("date", "dataCode","value", "period")], statement.types[d$statementType]) + d$statementType <- stypes[d$statementType] + + #partition and format output + tsubs <- split(d[, c("date", "dataCode","value", "period")], d$statementType) r <- lapply(tsubs, function(tsub) { dsubs <- split(tsub, tsub$period) #partition by period (Q or A), pivot and convert to a matrix @@ -75,7 +74,7 @@ getFinancials.tiingo <- function(Symbol, from, to, as.reported=FALSE, api.key, . if (NROW(dsub) < 1) return(NULL) pivot <- reshape(dsub[, c("date", "dataCode", "value")], timevar = "date", idvar = "dataCode", direction = "wide") - rownames(pivot) <- pivot[[1]] + rownames(pivot) <- pivot[[1]] #row names should be unique at this point. an assumption has been violated if not pivot <- pivot[, -1, drop = FALSE] colnames(pivot) <- gsub("^value\\.", "", colnames(pivot)) return(as.matrix(pivot)) From 30354190f814f3b385766ea75d31ad26000685d0 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Mon, 17 Jan 2022 16:11:11 -0700 Subject: [PATCH 10/15] cleanup cleanup --- R/getFinancials.R | 4 ++-- man/getFinancials.Rd | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index efb6ba1b..73b4eeea 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,4 +1,4 @@ -`getFinancials` <- getFin <- +getFin <- `getFinancials` <- function(Symbols, env=parent.frame(), src="tiingo", auto.assign=TRUE, from=Sys.Date()-720, to=Sys.Date(), ...) { #As much generic functionality and error handlign has been moved into the master function #source specific fucnimplementations should just fetch data for a single symbol and be as lightweight as possible @@ -31,7 +31,7 @@ return(unlist(ret.sym)) } -`viewFinancials` <- `viewFin` <- +`viewFin` <- `viewFinancials` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), subset = NULL) { importDefaults("viewFinancials") if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) diff --git a/man/getFinancials.Rd b/man/getFinancials.Rd index 6e0323dd..e9be002e 100644 --- a/man/getFinancials.Rd +++ b/man/getFinancials.Rd @@ -46,14 +46,12 @@ for quarterly. } \value{ Six individual matrices and a data.frame organized in a list of class \sQuote{financials}: - \item{ periods }{ a data.frame containing Fiscal year and quarter corresponding to each statement contained in the results. - This acts as a manifest for the contents of the rest of the structure. - The \code{ending} column matches the column name in the statement matrix corresponding to period. - } \item{ IS }{ a list containing (Q)uarterly and (A)nnual Income Statements } \item{ BS }{ a list containing (Q)uarterly and (A)nnual Balance Sheets } \item{ CF }{ a list containing (Q)uarterly and (A)nnual Cash Flow Statements } - + \item{periods}{a data.frame containing Fiscal year and quarter corresponding to each statement contained in the results. + The \code{date} column matches the column name in the statement matrix corresponding to period. Some sources moy not include this in the output} + Only the data structure itself is normalized. Individual statement matrixes use field names from the underlying source. Please refer to the source documentation for definitions and explanations. } From 63ef608e4965f9789e80c6378ef9dc10235ca4b5 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Mon, 17 Jan 2022 16:13:17 -0700 Subject: [PATCH 11/15] Update getFinancials.R --- R/getFinancials.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index 73b4eeea..efb6ba1b 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -1,4 +1,4 @@ -getFin <- `getFinancials` <- +`getFinancials` <- getFin <- function(Symbols, env=parent.frame(), src="tiingo", auto.assign=TRUE, from=Sys.Date()-720, to=Sys.Date(), ...) { #As much generic functionality and error handlign has been moved into the master function #source specific fucnimplementations should just fetch data for a single symbol and be as lightweight as possible @@ -31,7 +31,7 @@ getFin <- `getFinancials` <- return(unlist(ret.sym)) } -`viewFin` <- `viewFinancials` <- +`viewFinancials` <- `viewFin` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), subset = NULL) { importDefaults("viewFinancials") if(!inherits(x,'financials')) stop(paste(sQuote('x'),'must be of type',sQuote('financials'))) From 60457a4e786c9a02b6ccfd12fff9e996d452b710 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Tue, 18 Jan 2022 11:36:51 -0700 Subject: [PATCH 12/15] added conversion to data.frame --- R/getFinancials.R | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/R/getFinancials.R b/R/getFinancials.R index efb6ba1b..90e20230 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -52,6 +52,28 @@ return(t(as.xts(t(r))[subset])) } +as.data.frame.financials <- function(x) { + #reshape nested wide matrices to a long data.frame, adding columns for each nesting level + do.call("rbind", args = lapply(c("BS", "IS", "CF"), function(st) { #statement type loop + if(is.null(x[[st]])) return(NULL) + r <- do.call("rbind", lapply(c("A","Q"), function(p) { #period loop + #convert wide matrix to long dataframe + p.df <- as.data.frame(x[[st]][[p]]) + if (is.null(p.df) || nrow(p.df) < 1 || ncol(p.df) < 1 ) return(NULL) + cn <- colnames(p.df) + p.df <- reshape(p.df, direction = "long", varying = cn, times = cn, v.names = "value", ids = rownames(p.df)) + rownames(p.df) <- NULL + p.df$time <- as.Date(p.df$time) + p.df$period <- p + p.df <- p.df[, c("time", "id", "period", "value")] + colnames(p.df) <- c("date", "entry", "period", "value") + return(p.df) + })) + r$type <- st + return(r) + })) +} + getFinancials.tiingo <- function(Symbol, from, to, as.reported=FALSE, api.key, ...) { #API Documentation: https://api.tiingo.com/documentation/fundamentals importDefaults("getFinancials.tiingo") @@ -60,6 +82,7 @@ getFinancials.tiingo <- function(Symbol, from, to, as.reported=FALSE, api.key, . d <- suppressWarnings(read.csv(URL)) if (ncol(d) == 1 && colnames(d) == "None") stop("No data returned for Symbol: ", Symbol) + #reshape long dataframe to nested wide matrices, moving column into list elemnt names stypes <- c(balanceSheet = "BS", incomeStatement = "IS", cashFlow = "CF") d <- d[d$statementType %in% names(stypes) & d$quarter %in% (0:4),] d$period <- ifelse(d$quarter == 0, "A", "Q") @@ -73,7 +96,7 @@ getFinancials.tiingo <- function(Symbol, from, to, as.reported=FALSE, api.key, . lapply(dsubs, function(dsub) { if (NROW(dsub) < 1) return(NULL) pivot <- reshape(dsub[, c("date", "dataCode", "value")], - timevar = "date", idvar = "dataCode", direction = "wide") + timevar = "date", idvar = "dataCode", direction = "wide") rownames(pivot) <- pivot[[1]] #row names should be unique at this point. an assumption has been violated if not pivot <- pivot[, -1, drop = FALSE] colnames(pivot) <- gsub("^value\\.", "", colnames(pivot)) From 76d202ab2e537f9219bcc3a68c15d81734c83f63 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:58:34 -0700 Subject: [PATCH 13/15] docs on as.data.frame --- man/getFinancials.Rd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/man/getFinancials.Rd b/man/getFinancials.Rd index e9be002e..f1792016 100644 --- a/man/getFinancials.Rd +++ b/man/getFinancials.Rd @@ -3,6 +3,7 @@ \alias{viewFinancials} \alias{getFin} \alias{viewFin} +\alias{as.data.frame.financials} \title{ Download and View Financial Statements } \description{ Download Income Statement, Balance Sheet, and Cash Flow Statements. @@ -43,6 +44,8 @@ sheet, IS for income statement, and CF for cash flow statement. The period argument is used to identify which statements to view - (A) for annual and (Q) for quarterly. + +\code{as.data.frame.financials} unpacks the finacials list into a wide formate data.frame } \value{ Six individual matrices and a data.frame organized in a list of class \sQuote{financials}: From d4fca2b6df9c42258df662264229b5fada2c4e24 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Wed, 19 Jan 2022 15:49:04 -0700 Subject: [PATCH 14/15] added as.data.frame test --- tests/test_getFinancials.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_getFinancials.R b/tests/test_getFinancials.R index 7aa101cb..55b7f1f3 100644 --- a/tests/test_getFinancials.R +++ b/tests/test_getFinancials.R @@ -9,6 +9,10 @@ if (nzchar(apikey)) { aapl <- getFinancials("AAPL", src = "tiingo", api.key = apikey, auto.assign = FALSE) stopifnot(inherits(aapl, "financials")) + aapl.df <- as.data.frame(aapl) + stopifnot(is.data.frame(aapl.df)) + #conversion to df shoudl test unpacing nested elements + stopifnot(names(aapl.df) == c("date", "entry", "period", "value", "type")) #test multisymbol path with bad symbols retsym <- getFinancials("AAPL;BA;UNKNOWNSYMBOL", src = "tiingo", From d331c0582ca406b6f8ded914339f69ca517a0f49 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Wed, 19 Jan 2022 15:56:10 -0700 Subject: [PATCH 15/15] Update getFinancials.R --- R/getFinancials.R | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/R/getFinancials.R b/R/getFinancials.R index 90e20230..59b5150e 100644 --- a/R/getFinancials.R +++ b/R/getFinancials.R @@ -31,6 +31,21 @@ return(unlist(ret.sym)) } +getFinancials.google <- +function(Symbol, env=parent.frame(), src="google", auto.assign=TRUE, ...) { + msg <- paste0(sQuote("getFinancials.google"), " is defunct.", + "\nGoogle Finance stopped providing data in March, 2018.", + "\nYou could try some of the data sources via Quandl instead.", + "\nSee help(\"Defunct\") and help(\"quantmod-defunct\")") + .Defunct("Quandl", "quantmod", msg = msg) +} + +`print.financials` <- function(x, ...) { + cat('Financial Statement for',attr(x,'symbol'),'\n') + cat('Retrieved from',attr(x,'src'),'at',format(attr(x,'updated')),'\n') + cat('Use "viewFinancials" or "viewFin" to view\n') +} + `viewFinancials` <- `viewFin` <- function(x, type=c('BS','IS','CF'), period=c('A','Q'), subset = NULL) { importDefaults("viewFinancials")