/*
Purpose: Clean power_kind* variables in 1850 data.
*/


* Set file paths based on username
if "`1'"==""{
	global CMF_pipeline "../.."
}
* Define directory globals
global input "${CMF_pipeline}/5_industry_assignment/output"
global interm "${CMF_pipeline}/6_power_machine_cleaning/intermediate_files"
global output "${CMF_pipeline}/6_power_machine_cleaning/output"


* -----------------------------
* STEP 1: Import cleaning crosswalk
* -----------------------------
import excel "$interm\power_machine_category.xlsx", sheet("Sheet1") firstrow clear
capture drop freq done
rename ctype power_cleaned_kind
rename ctype_steam power_steam
rename ctype_water power_water
rename ctype_wind power_wind
rename ctype_horse power_horse
rename ctype_hand power_hand
rename ctype_machine machine_dummy
rename ctype_machine_category machine_category
tempfile cleaned_strings
save `cleaned_strings', replace


* -----------------------------
* STEP 2: Clean raw power_kind* fields (from 1850 industry-assigned data)
* -----------------------------
use "${input}\1850_industry_assigned.dta", clear

* Clean and merge each raw power_kind variable
foreach j of numlist 1/11 14 {
	rename power_kind`j' power_kind_raw
	replace power_kind_raw = lower(power_kind_raw)
	merge m:1 power_kind_raw using "$interm\power_type_cleaned.dta"
	drop if _merge == 2
	cap drop _merge

	* Save cleaned results with consistent naming
	foreach i of numlist 1/11 {
		rename cleaned_unit_`i' power_cleaned_unit`j'_`i'
		rename cleaned_type_`i' power_cleaned_kind`j'_`i'
	}
	rename power_kind_raw power_kind`j'
}
keep file_name firm_number power_cleaned*


* -----------------------------
* STEP 3: Drop empty cleaned variables
* -----------------------------
foreach var of varlist power_cleaned_kind* {
	capture assert mi(`var')
	if !_rc {
		drop `var'
		local unit_var = substr("`var'",19,.)
		local unit_var = "power_cleaned_unit"+"`unit_var'"
		drop `unit_var'
	}
}


* -----------------------------
* STEP 4: Re-index suffix numbers
* -----------------------------
local i = 1
foreach var of varlist power_cleaned_unit* {
	rename `var' power_cleaned_unit`i'
	local i = `i'+1
}
local i = 1
foreach var of varlist power_cleaned_kind* {
	rename `var' power_cleaned_kind`i'
	local i = `i'+1
}


* -----------------------------
* STEP 5: Reshape to long format
* -----------------------------
sort file_name firm_number
reshape long power_cleaned_unit power_cleaned_kind, i(file_name firm_number) j(iter)
drop if power_cleaned_kind == ""


* -----------------------------
* STEP 6: Merge cleaned power types and create dummies
* -----------------------------
merge m:1 power_cleaned_kind using `cleaned_strings'
drop if _merge == 2
gsort file_name firm_number iter
cap drop _merge iter

foreach var of varlist power_steam power_water power_wind power_horse power_hand {
	replace `var' = 0 if missing(`var')
}

tempfile power_machine_1850
save `power_machine_1850', replace


* -----------------------------
* STEP 7: Collapse to establishment-level power use
* -----------------------------
use `power_machine_1850', clear
gen any_fire = strpos(power_cleaned_kind, "fire")>0
keep file_name firm_number power_steam power_water power_wind power_horse power_hand any_fire
collapse (max) power_steam power_water power_wind power_horse power_hand any_fire, by(file_name firm_number)
tempfile power_1850
save `power_1850', replace


* -----------------------------
* STEP 8: Extract machine use
* -----------------------------
use `power_machine_1850', clear
keep file_name firm_number machine_dummy power_cleaned_unit power_cleaned_kind machine_category
keep if machine_dummy == 1
cap drop machine_dummy
bys file_name firm_number: gen iter = _n
rename (power_cleaned_unit power_cleaned_kind machine_category) (machine_unit machine_kind machine_category)
reshape wide machine_unit machine_kind machine_category, i(file_name firm_number) j(iter)
tempfile machine_1850
save `machine_1850', replace


* -----------------------------
* STEP 9: Merge power and machine data back to 1850 dataset
* -----------------------------
use "${input}\1850_industry_assigned.dta", clear
merge 1:1 file_name firm_number using `power_1850', nogen

* Ensure power dummies are not missing
foreach var of varlist power_steam power_water power_wind power_horse power_hand {
	replace `var' = 0 if missing(`var')
}

merge 1:1 file_name firm_number using `machine_1850'
drop _merge


* -----------------------------
* STEP 10: Industry-based corrections
* Flag steam/water power for mills when indicated in industry string
* -----------------------------
replace power_steam = 1 if strpos(industry_raw,"steam") > 0 & inlist(ind_leontief, "lumber", "flour and grist mills")
replace power_water = 1 if strpos(industry_raw,"water") > 0 & inlist(ind_leontief, "lumber", "flour and grist mills")
replace power_steam = 1 if any_fire == 1 & inlist(ind_leontief, "lumber", "flour and grist mills")
drop any_fire


* -----------------------------
* STEP 11: Save final cleaned dataset
* -----------------------------
save "$output\1850_power_machine_vars_cleaned.dta", replace