/*
Author: Vivian Zhang
Date: 01/20/2021
Purpose: Reshape the data so that each establishment only occupies one line

MODIFIED BY ALLEN OTTERSTROM 1/30/2025
*/

if "`c(username)'"=="otterstrom"{
	global CMF_pipeline "C:\Users\otterstrom\UChicago Dropbox\Allen Otterstrom\CMF Micro Data\2_clean_pipeline"
}
else {
	global CMF_pipeline "C:\Users\`c(username)'\Dropbox (UChicago)\CMF Micro Data\2_clean_pipeline"
}
global input "${CMF_pipeline}\2_string_cleaning\output"
global output "${CMF_pipeline}\3_reshape\output"
global intermediate "${CMF_pipeline}\3_reshape\interm"
ssc install missings, replace
foreach y in 1850 1860 1870{
	use "$input/CMF_`y'.dta", clear

	//Drop totals, fips since  rows
	//drop if totals != "0" 
	*Drop anything uneeded for the reshape
	drop totals 
	
    //Drop duplicate rows that are the same in every variable
    duplicates drop
    //Fix identical firm lines within the same file_name and firm_number 
	sort file_name firm_number firm_line
    replace firm_line = 1 if firm_line == 0
    replace firm_line = firm_line[_n-1] +1 if firm_line == firm_line[_n-1] & file_name == file_name[_n-1] & firm_number == firm_number[_n-1]
    replace firm_line = firm_line[_n-1] +1 if firm_line != firm_line[_n-1] + 1 & file_name == file_name[_n-1] & firm_number == firm_number[_n-1]
    
    if `y' == 1850 | `y' == 1860 {
		//Separate certain columns prior to reshaping to save reshaping run-time
        preserve
        keep file_name firm_number firm_line comment post_office county state id_number multiple_counties firm_name capital hands_male hands_female ///
        avg_wage_male avg_wage_female industry_raw industry maker shop manufacture factory
        save "$intermediate/`y'_separated_vars.dta", replace
        restore
        drop comment post_office county state id_number multiple_counties firm_name capital hands_male hands_female ///
        avg_wage_male avg_wage_female industry_raw industry maker shop manufacture factory
		
        reshape wide materials_qty materials_unitofmeasure materials_kinds_raw materials_value power_kind prod_qty prod_unitofmeasure production_kinds_raw ///
        production_values product_is_note product_misc product_service product_is_units product_unsure product_comments material_is_note material_misc ///
        material_service material_units material_unsure material_comments material_kind product_kind material_kind_attr product_kind_attr, i(file_name firm_number) j(firm_line)
        }
    if `y' == 1870 {
		//Separate certain columns prior to reshaping to save reshaping run-time
        preserve
        keep file_name firm_number firm_line comment post_office county state id_number multiple_counties firm_name capital hands_male hands_female ///
        hands_children tot_wages industry_raw industry maker shop manufacture factory months_active
        save "$intermediate/`y'_separated_vars.dta", replace
        restore
        drop comment post_office county state id_number multiple_counties firm_name capital hands_male hands_female hands_children tot_wages industry_raw ///
		industry maker shop manufacture factory to_keep dup dup2 dup_group obs months_active
		*Added
		
		
        reshape wide power_kind horsepower_number materials_kinds_raw materials_qty materials_unitofmeasure ///
        materials_value production_kinds_raw prod_qty prod_unitofmeasure production_values product_is_note product_misc product_service product_is_units product_unsure ///
        product_comments material_is_note material_misc material_service material_units material_unsure material_comments material_kind product_kind ///
        material_kind_attr product_kind_attr machine_description machine_number, i(file_name firm_number) j(firm_line)
        }
    missings dropvars, force
	compress
    save "$output/`y'_reshaped.dta", replace

	//Merge the separated columns back to the reshaped data
    use "$intermediate/`y'_separated_vars.dta"
    replace firm_line=1 if firm_number!=firm_number[_n-1]& firm_line!=1
    keep if firm_line == 1
    drop firm_line
    compress
    save "$intermediate/`y'_vars_merge.dta", replace
    use "$output/`y'_reshaped.dta"
    merge 1:1 file_name firm_number using"$intermediate/`y'_vars_merge.dta"
    drop _merge
	
	replace file_name = ustrtrim(ustrnormalize(file_name, "nfd"))
	sort file_name firm_number
    save, replace
    
}