/*************************************************************************************************************
This do-file reads in Eurostat data. The data are downloaded from the bulk download facility: http://epp.eurostat.ec.europa.eu/NavTree_prod/everybody/BulkDownloadListing 

Written by Gabriel Chodorow-Reich and provided without any warranty
This version: February 2017

Sample syntax: 


*************************************************************************************************************/

capture program drop loadeurostat
program define loadeurostat
version 12.1
syntax anything(name=dataset id="Name of data set to load"), [Ivariables(namelist) Jvariable(name) FREQuency(name) log(string) vintage(string) local(string) saving(string) noclean] 

clear all
set more off 

/*************************************************************************************************************
Preliminaries
*************************************************************************************************************/

if `"`log'"'!=`""' {
	cap log close
	log using `log', replace
}
if regexm(`"`frequency'"',`"^[HMQYhmqy][a-z]+$"') {
	local f = lower(substr(`"`frequency'"',1,1))
	local F = upper(`"`f'"')
}
if `"`vintage'"'!=`""' local vintageprefix https://web.archive.org/web/`vintage'/

/*************************************************************************************************************
Read in data
*************************************************************************************************************/
if `"`local'"'==`""' copy "`vintageprefix'https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&downfile=data%2F`dataset'.gz" `dataset'.gz, replace
else copy `"`local'"' `dataset'.gz, replace
gunzipfile `dataset'.gz, replace
import delimited using `dataset', delimiters("\t") clear asdouble varnames(nonames)
erase `dataset'
erase `dataset'.gz

if `"`clean'"'==`"noclean"' {
	exit
}

rename v1 codes
foreach var of varlist v* {
	local varname = `var'[1]
	rename `var' value`varname'
}
if regexm(codes[1],`"(\\)([a-z]+)"') local colvar = regexs(2)
replace codes = regexr(codes,`"\\[a-z]+"',`""') in 1
split codes, parse(,) gen(labs)
drop codes
foreach lab of varlist labs* {
	local varname = `lab'[1]
	ren `lab' `varname'
	local ivariablesifmissing `"`ivariablesifmissing' `varname'"'
}
if `"`ivariables'"'==`""' local ivariables = subinstr(`"`ivariablesifmissing'"',`"`jvariable'"',`""',1)
drop in 1
qui destring value*, force replace ignore(" " "e" "p" "b")
local reshapei = regexr(`"`ivariables' `jvariable'"',`"`colvar'"',`""')
reshape long value, i(`reshapei') j(`colvar') string
if `"`colvar'"'==`"time"' {
	if inlist(`"`frequency'"',"annual","year","yearly") {
		destring `colvar', gen(year)
		local frequency year
	}
	else {
		gen `frequency' = `frequency'(`colvar',"Y`F'") 
		format `frequency' %t`f'
	}
	drop `colvar'
}
else {
	cap ren time `frequency'
	cap destring `frequency', replace
	local ivariables = subinstr(`"`ivariables'"',`"time"',`""',1)
}

/*************************************************************************************************************
Documentation files
*************************************************************************************************************/
foreach doc in `ivariables' `jvariable' {
	preserve
	import delimited using "https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=dic%2Fen%2F`doc'.dic", delimiters("\t") clear
	if inlist(`"`doc'"',`"`jvariable'"') {
		forvalues i=1/`=_N' {
			local `=substr(`"`=v1[`i']'"',1,30)' `"`=v2[`i']'"'
		}
		restore
	}
	else {
		rename v1 `doc'
		rename v2 `doc'_label
		cap tostring `doc', replace
		tempfile data
		save `data', replace
		restore
		merge m:1 `doc' using `data', keep(master matched) nogenerate
		local order `"`order' `doc'_label"'
	}
}

/*************************************************************************************************************
Format and save
*************************************************************************************************************/
if `"`jvariable'"'!=`""' {
	compress
	levelsof `jvariable', local(levels) clean
	reshape wide value, i(`ivariables' `frequency') j(`jvariable') string
	ren value* *
	foreach var of local levels {
		label variable `var' `"``var''"'
		local dropifmissing `"`dropifmissing' & missing(`var')"'
	}
	di `"drop if 1==1 `dropifmissing'"'
	drop if 1==1 `dropifmissing'
}

di `"order `order' `frequency' `levels'"'
order `order' `frequency' `levels'
di `"sort `ivariables' `frequency'"'
sort `ivariables' `frequency'
compress

if `"`saving'"'!=`""' {
	cap drop __00*
	local filename = regexr(`"`saving'"',`"^.*[/\\]"',`""')
	save `"`filename'"', replace
	desc
	gzipfile `"`filename'.dta"', saving(`"`saving'"') replace erase `mac'	
}

cap log close

end