/*************************************************************************************************************
This do-file reads in European bond yields. The data are downloaded from the bulk download facility: http://epp.eurostat.ec.europa.eu/NavTree_prod/everybody/BulkDownloadListing 

1. Read in data
2. Merge with documentation files
3. Save

Written by Gabriel Chodorow-Reich and provided without any warranty
This version: June 2016

Sample syntax: dgs10, saving("$SavePath/european bond yields") gzip replace

*************************************************************************************************************/

capture program drop dgs10
program define dgs10
version 12.1
syntax, [saving(string)] [gzip] [zip] [replace]

clear all
#delimit;
set more off; 

tempfile data;
set maxvar 12000;
capture copy https://dl.dropboxusercontent.com/s/v5f4ob7bl10ffmo/gunzipfile.ado;
capture copy https://dl.dropboxusercontent.com/s/8bxqrcknhh57k2w/gzipfile.ado;
/*************************************************************************************************************
1. Read in data
*************************************************************************************************************/
local rawdata irt_lt_mcby_d.tsv;
copy "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=data%2F`rawdata'.gz" `rawdata'.gz, replace;
gunzipfile `rawdata'.gz, replace;
qui insheet using `rawdata', tab clear double;
qui erase `rawdata';
qui erase `rawdata'.gz;

qui egen data_code = ends(v1), punct(",") head;
local data_code = data_code[2];
qui egen region_code = ends(v1), punct(",") tail;
qui drop v1;
forvalues n = 1/`=_N' {;
	local region_code`n' `"`=region_code[`n']'"';
};
drop data_code region_code;
foreach var of varlist v* {;
	qui replace `var' = string(date(`var'[1], "Y#M#D")) in 1; /*Format date in stata format*/
};
mata : D = st_sdata(.,.); /*Creates Mata string matrix D*/
mata: D_n = strtoreal(D); /*Destring matrix D*/
mata: D_n = D_n'; /*Transpose data*/
clear;
getmata (v*)= D_n;
ren v1 daten;
qui format daten %td;
foreach var of varlist v* {; /*Reapply variable names*/
	local vn = substr(`"`var'"',2,.);
	ren `var' DGS10`region_code`vn'';
	qui format DGS10`region_code`vn'' %9.2f;
};

/*************************************************************************************************************
2. Documentation files
*************************************************************************************************************/
preserve;
qui insheet using "http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=dic%2Fen%2Fgeo.dic", tab clear;
local N = _N;
forvalues i = 1/`N' {;
	local v`i' = v1[`i']; *Local name is region abbreviation;
	local `v`i'' = v2[`i']; *Local value is region full name;
};
qui levelsof v1, local(regions);
restore;

/*************************************************************************************************************
3. Format and save
*************************************************************************************************************/
foreach r of local regions {;
	capture label variable DGS10`r' "``r''";
};
qui tsset date;
gen data_code = `"`data_code'"';

if `"`saving'"'!=`""' {;
	if `"`replace'"'==`"replace"' {;
		local replace `", replace"';
	};
	if `"`zip'"'==`"zip"' | `"`gzip'"'==`"gzip"' {;
		local filename = regexr(`"`saving'"',`"^.*[/\\]"',`""');
		qui save `"`filename'"', replace;
		qui `zip'`gzip'file `"`filename'.dta"', saving(`"`saving'"' `replace');
		erase `"`filename'.dta"';
	};
	else {;
		qui save `"`saving'"' `replace';
	};
};


end;