/*************************************************************************************************************
This do-file reads data from the Financial Accounts of the United States (FAUS) from:
http://www.federalreserve.gov/releases/z1/Current/data.htm 

Options:
Saving: Path to save data set
Vintage: If unspecified, will read in current vintage. Otherwise, must be CCYYMMDD format for a day of the FoF release, i.e. 20110916.

Sample command: loadfof, saving("$SavePath/fof") zip replace

Written by Gabriel Chodorow-Reich and provided without any warranty
This version: October 2016

*************************************************************************************************************/

capture program drop loadfof
program define loadfof
version 12.1
syntax, [saving(string) zip replace vintage(string)]

/*************************************************************************************************************
Load data
*************************************************************************************************************/
if `"`vintage'"'==`""' qui copy https://www.federalreserve.gov/releases/z1/current/z1_csv_files.zip z1_csv_files.zip, replace
else qui copy https://www.federalreserve.gov/releases/z1/`vintage'/z1_csv_files.zip z1_csv_files.zip, replace
qui unzipfile z1_csv_files.zip, replace
erase z1_csv_files.zip
local tables: dir csv files "*"
foreach t of local tables {
	local tname = regexr(`"`t'"',`"\.csv$"',`""')
	di `"`tname'"'
	
	/*Meta data*/
	qui import delimited using data_dictionary/`tname'.txt, clear
	ren v1 code
	ren v2 description
	ren v3 line
	ren v4 table 
	ren v5 units
	qui replace line = regexr(line,"^Line ","")
	qui destring line, force replace
	qui replace code = regexr(code,`"\."',`""')
	qui duplicates drop code, force
	qui replace code = regexr(code,`"Q$"',`"A"') if inlist(`"`tname'"',`"b101h"',`"b101n"') /*Mis-coding in dictionary files*/
	tempfile dictionary
	qui save `dictionary'
	
	/*Actual data*/
	qui import delimited using csv/`tname'.csv, clear varnames(1) case(preserve) stringcols(_all)
	cap drop v* /*Duplicates in all sector tables*/
	ren * value*
	ren valuedate date
	qui reshape long value, i(date) j(code) string
	qui destring value, ignore("ND") replace
	if regexm(code[1],`"A$"') { /*Annual frequency*/
		ren date year
	}
	else if regexm(code[1],`"Q$"') { /*Quarterly frequency*/			
		gen int quarterly = quarterly(date,"YQ")
		qui format quarterly %tq
		drop date
		gen year = year(dofq(quarterly))
	}
	cap confirm numeric year
	if _rc!=0 qui destring year, force replace
	
	/*Merge meta data*/
	qui merge m:1 code using `dictionary', assert(matched) nogenerate noreport
	erase data_dictionary/`tname'.txt
	erase csv/`tname'.csv
	
	/*Append*/
	if `"`notfirst'"'==`""' {
		local notfirst notfirst
		tempfile data
	}
	else qui append using `data'
	qui save `data', replace
}

 
/*************************************************************************************************************
4. Clean and save
*************************************************************************************************************/
ren units units_string
qui encode units_string, gen(units)
drop units_string
order year quarterly code table line value description units
sort table year quarterly line

#delimit;

if `"`saving'"'!=`""' {;
	if `"`zip'"'==`"zip"' {;
		local filename = regexr(`"`saving'"',`"^.*[/\\]"',`""');
		qui save `"`filename'"', replace;
		qui zipfile `"`filename'.dta"', saving(`"`saving'"', `replace');
		erase `"`filename'.dta"';
	};
	else qui save `"`saving'"', `replace';
};


end;