/*************************************************************************************************************
This ado file reads in the CPS basic monthly files, 1989-present, from https://www2.census.gov/programs-surveys/cps/datasets/ and using the NBER programs from http://www.nber.org/data/cps_basic_progs.html.

Options:
savepath: location to save data set
nber: download all data sets from NBER repository instead of from Census

Example: to download CPS basic monthly files for 1990-2013 and save the resulting data sets in myfolder, type into command line
load_cpsbasic, year0(1990) year1(2013) month0(1) month1(12) savepath(myfolder).
load_cpsbasic `=tm(2020m1)'/`=tm(2020m4)', savepath($DropboxPath/data/dta/cps/basic) programdirectory(programs).

Notes: the NBER files for 1996 and 1997 appear to have the same data for all months in the year. The corrupted Census file for December 1995 has been replaced with a valid file.
However, the Census December 1995 file contains 21 duplicate observations on the variables used to match across months.

*************************************************************************************************************/

capture program drop load_cpsbasic
program define load_cpsbasic
version 12.0
syntax [anything], [year0(integer 1976) year1(integer 1976) month0(integer 1) month1(integer 1) nber programdirectory(string) savepath(string)]
#delimit;
set more off;
if `"`anything'"'==`""' local anything `=ym(`year0',`month0')'(1)`=ym(`year1',`month1')';
if `"`anything'"'==`""' {;
	di as error `"Either month list required as main argument or year0,year1,month0,month1 required as options."';
	exit as error;
};
numlist `"`anything'"', integer sort;

/*************************************************************************************************************
0. Check for program directory
*************************************************************************************************************/
if `"`programdirectory'"'!=`""' {;
	local cdir `"`c(pwd)'"';
	cap cd `"`programdirectory'"';
	if _rc!=0 {;
		cap mkdir `"`programdirectory'"';
		if _rc!=0 {;
			di as error "Program directory cannot be created";
			exit;
		};
	};
	cd `"`cdir'"';	
};
if `"`:dir `"`programdirectory'"' files `"*"' '"'==`""' {;
	copy https://www.dropbox.com/s/pkrbvq1emayk8hj/cpsbjan2015.dct?dl=0 `"`programdirectory'/cpsbjan2015.dct"';
	copy https://www.dropbox.com/s/x3qku5gqucgqonw/cpsbjan2015.do?dl=0 `"`programdirectory'/cpsbjan2015.do"';
	copy https://www.dropbox.com/s/yxh2a0sxoec6j52/cpsbjan2017.dct?dl=0 `"`programdirectory'/cpsbjan2017.dct"';
	copy https://www.dropbox.com/s/5ub043ie3ki7ele/cpsbjan2017.do?dl=0 `"`programdirectory'/cpsbjan2017.do"';
	copy https://www.dropbox.com/s/dt14kfkqf8zmd8q/cpsbjan2020.dct?dl=0 `"`programdirectory'/cpsbjan2020.dct"';
	copy https://www.dropbox.com/s/1a6u532d9leagng/cpsbjan2020.do?dl=0 `"`programdirectory'/cpsbjan2020.do"';
};

/*************************************************************************************************************
1. Read in data
*************************************************************************************************************/
foreach monthly in `r(numlist)' {;
	local yyyy = year(dofm(`monthly'));
	local m = month(dofm(`monthly'));
	local yy = substr(`"`yyyy'"',3,2);
	local mon = string(mdy(`m',1,`yyyy'),`"%tdmon"');	
	clear all;
	di "`mon'`yy'";
	global filename `mon'`yy'pub;
	
	*Assign relevant data dictionary;
	foreach dictionarymonth in jan89 jan92 jan94 apr94 jun95 sep95 jan98 jan03 may04 aug05 jan07 jan09 jan10 may12 jan13 jan2014 apr2014 jan2015 jan2017 jan2020 jan2023 jan2024 {;
		local dictionary_monthly = monthly("`dictionarymonth'","MY",2050); /*Month of dictionary in stata format*/
		local dictionary_month_m = month(dofm(`dictionary_monthly')); /*Calendar month of dictionary*/
		if length(`"`dictionary_month_m'"')==2 {;
			local mm = `dictionary_month_m';
		};
		else {;
			local mm `"0`dictionary_month_m'"';
		};
		if `monthly'<`dictionary_monthly' {;
			/*From 1994-1997 census path includes the range of months over which the dictionary is valid - thereafter it includes only the dictionary month.*/
			local dictionary_monthly1 = `dictionary_monthly'-1;
			local dictionary_month_m1 = month(dofm(`dictionary_monthly1')); /*Calendar month of dictionary*/
			if length(`"`dictionary_month_m1'"')==2 {;
				local mm1 = `dictionary_month_m1';
			};
			else {;
				local mm1 `"0`dictionary_month_m1'"';
			};				
			local census_dictionarymonth1 `=year(dofm(`dictionary_monthly1'))'`mm1';		
			continue, break;
		};
		local census_dictionarymonth `=year(dofm(`dictionary_monthly'))'`mm';
		if `monthly'<tm(2023m1) local nber_dictionarymonth `dictionarymonth';
		else local nber_dictionarymonth `census_dictionarymonth';
		if inrange(`monthly',tm(2014m4),tm(2014m12)) {; /*Census did not change path name to reflect April 2014 dictionary*/
			local census_dictionarymonth 201401;
		};
	};
	di `"`nber_dictionarymonth'"';
	di `"`census_dictionarymonth'"';

	*Download data;
	if `monthly'>=tm(1994m1) {;
		if `monthly'<tm(1998m1) & `"`nber'"'==`""' {;
			qui copy https://thedataweb.rm.census.gov/pub/cps/basic/`census_dictionarymonth'-`census_dictionarymonth1'/$filename.zip $filename.zip, replace;
		};
		else if `"`nber'"'==`""' & `monthly'<tm(2020m1) {;
			qui copy https://thedataweb.rm.census.gov/pub/cps/basic/`census_dictionarymonth'-/$filename.zip $filename.zip, replace;
		};
		else if  `"`nber'"'==`""' {;
			qui copy https://www2.census.gov/programs-surveys/cps/datasets/`yyyy'/basic/$filename.zip $filename.zip, replace;
		};
		else {;
			qui copy http://www.nber.org/cps-basic/$filename.zip $filename.zip, replace;
		};
		qui unzipfile $filename.zip, replace;
		*Name of data file and extension changes. I create a macro with the name of the data file.;
		local datfile: dir "" files "*.dat";
		foreach ext in cps pub {;
			if `"`datfile'"'=="" {;
				local datfile: dir "" files "*.`ext'";
			};
		};		
	};
	else if inrange(`monthly',tm(1989m1),tm(1993m12)) {; /*1989-93 files provided by David Card and with different naming convention*/
		if `m'<10 {;
			global Cardfilename cpsb`yy'0`m';
		};
		else {;
			global Cardfilename cpsb`yy'`m';
		};
		qui copy http://nber.org/cps-basic/$Cardfilename.Z $Cardfilename.Z, replace;
		gunzip $Cardfilename, ext(Z);
		copy $Cardfilename $filename.txt;
		erase $Cardfilename.Z;	
		erase $Cardfilename;
		local datfile $filename.txt;
	};
	
	*Download program and dictionary;
	if `monthly'<tm(2017m1) {;
		qui copy http://www.nber.org/data/progs/cps-basic/cpsb`nber_dictionarymonth'.do cpsb`nber_dictionarymonth'.do, replace;
		qui copy http://www.nber.org/data/progs/cps-basic/cpsb`nber_dictionarymonth'.dct cpsb`nber_dictionarymonth'.dct, replace;			
	};
	else {;
		qui copy `programdirectory'/cpsb`nber_dictionarymonth'.do cpsb`nber_dictionarymonth'.do, replace;
		qui copy `programdirectory'/cpsb`nber_dictionarymonth'.dct cpsb`nber_dictionarymonth'.dct, replace;					
	};
	!type cpsb`nber_dictionarymonth'.do | findstr /v "local dat_name" | findstr /v "quietly infile" | findstr /v "save" | findstr /b /v "clear" >cpsb`nber_dictionarymonth'x.do;
	*Removes lines with specified strings. The paths in Jean Roth's files are MAC paths, and I am too lazy to change them by hand.;

	*Infile and save;
	qui infile using cpsb`nber_dictionarymonth'.dct, using(`datfile') clear;
	erase `datfile';
	do cpsb`nber_dictionarymonth'x;
	qui save $filename, replace;
	if `"`savepath'"'!=`""' & !regexm(`"`savepath'"',`"/$"') {;
		local savepath `"`savepath'/"';
	};
	zipfile $filename.dta, saving(`"`savepath'$filename"', replace);
	capture log close;
	foreach ext in .dta .zip .Z {;
		capture erase $filename`ext';
	};
	foreach ext in .do x.do .dct .log .smcl {;
		capture erase cpsb`nber_dictionarymonth'`ext';
	};
};


end;