/*************************************************************************************************************
This ado-file loads NIPA data from .

Options:
saving: location to save data set
log: log path

Example:
loadnipa annual, saving($SavePath/nipa-annual) log(log/nipa-annual_`=string(date(`"$S_DATE"',`"DMY"'),`"%tdCCYYNNDD"')'.log) verbose gzip 
loadnipa quarterly, saving($SavePath/nipa-quarterly) log(log/nipa-quarterly_`=string(date(`"$S_DATE"',`"DMY"'),`"%tdCCYYNNDD"')'.log) verbose gzip 
loadnipa monthly, saving($SavePath/nipa-monthly) log(log/nipa-monthly_`=string(date(`"$S_DATE"',`"DMY"'),`"%tdCCYYNNDD"')'.log) verbose gzip

*************************************************************************************************************/

capture program drop loadnipa
program define loadnipa
version 12.1
syntax anything(name=frequency id=`"annual or quarterly or monthly"'), [saving(string) zip gzip log(string) verbose]
set more off

/*************************************************************************************************************
Preliminaries
*************************************************************************************************************/
if `"`log'"'!=`""' {
	cap log close
	log using `log', replace
}
local f = substr(`"`frequency'"',1,1)
local F = upper(`"`f'"')
if `"`verbose'"'==`""' local qui qui

/*************************************************************************************************************
Load documentation
*************************************************************************************************************/
`qui' import delimited using https://apps.bea.gov/national/Release/txt/SeriesRegister.txt, clear varnames(1)
tempfile series
qui save `series'

`qui' import delimited using https://apps.bea.gov/national/Release/txt/TablesRegister.txt, clear varnames(1)
tempfile tables
qui save `tables'

/*************************************************************************************************************
Load data
*************************************************************************************************************/
`qui' import delimited using https://apps.bea.gov/national/Release/txt/NipaData`F'.txt, clear varnames(1)

/*************************************************************************************************************
Merge documentation
*************************************************************************************************************/
`qui' merge m:1 seriescode using `series', keep(matched) nogenerate 
* assert(using matched)
/*Add one observation per table/line/period*/
`qui' split tableidlineno, parse(|)
`qui' gen tableid = ""
`qui' gen lineno = ""
`qui' desc tableidlineno? tableidlineno??, varlist
gen byte original=1
`qui' forvalues i = 1/`:word count `r(varlist)'' {
	expand 2 if original, gen(newobs) /*One potential observation per table/line/period*/
	`qui' replace original = 0 if newobs
	`qui' split tableidlineno`i' if newobs, parse(:) gen(tempvar)
	`qui' replace tableid = tempvar1 if newobs
	`qui' replace lineno = tempvar2 if newobs
	drop if tableid == "" & newobs /*Drop if series not in `i' tables*/
	drop newobs tableidlineno`i' tempvar?
}
drop original

`qui' merge m:1 tableid using `tables', keep(matched) nogenerate
* assert(using matched)

/*************************************************************************************************************
Save
*************************************************************************************************************/
`qui' destring value, replace ignore(",")
ren tabletitle table
gen id_s = table + ": " + lineno + ": " + seriescode 
encode id_s, gen(id)
drop id_s
if inlist(`"`f'"',`"m"',`"q"') {
	gen int `frequency' = `frequency'(period,`"Y`F'"')
	qui format `frequency' %t`f'
	local date `frequency'
	drop period
}
else {
	ren period year
	local date year
}
`qui' destring lineno, replace
`qui' compress
order seriescode table lineno serieslabel `date' value metricname defaultscale
tsset id `date'

if `"`saving'"'!=`""' {
	label data `"Downloaded `=string(date(`"$S_DATE"',`"DMY"'),`"%tdMonth_dd,_CCYY"')'"'
	if `"`zip'"'==`""' & `"`gzip'"'==`""' `qui' save `"`saving'"', replace
	else {
		local filename = regexr(`"`saving'"',`"^.*/"',`""')
		local filename = regexr(`"`filename'"',`"^.*\\"',`""')
		`qui' save `"`filename'"', replace
		if `"`zip'"'!=`""' {
			qui zipfile `"`filename'.dta"', saving(`"`saving'"', replace) 
			erase `"`filename'.dta"'
		}
		else qui gzipfile `"`filename'.dta"', saving(`"`saving'.zip"') replace erase 
	}
}
if `"`log'"'!=`""' {
	desc
	log close
}

end