/*
Purpose: Clean power_kind* in 1860 data.
*/


if "`1'"==""{
	global CMF_pipeline "../.."
}

* Define folder globals for input, intermediate, and output
global input "${CMF_pipeline}/5_industry_assignment/output"
global interm "${CMF_pipeline}/6_power_machine_cleaning/intermediate_files"
global output "${CMF_pipeline}/6_power_machine_cleaning/output"

* ------------------------------------------------------------
* STEP 1. Import mapping between raw and cleaned power/machine strings
* ------------------------------------------------------------
import excel "$interm\power_machine_category.xlsx", sheet("Sheet1") firstrow clear
capture drop freq done        // drop extra columns if they exist
rename ctype power_cleaned_kind
rename ctype_steam power_steam
rename ctype_water power_water
rename ctype_wind power_wind
rename ctype_horse power_horse
rename ctype_hand power_hand
rename ctype_machine machine_dummy
rename ctype_machine_category machine_category
tempfile cleaned_strings
save `cleaned_strings', replace   // save cleaned mapping as temp file

* ------------------------------------------------------------
* STEP 2. Merge raw power_kind* variables with cleaned mapping
* ------------------------------------------------------------
use "${input}\1860_industry_assigned.dta", clear
foreach j of numlist 1/11 14 {             // loop through power_kind1–11 and power_kind14
	rename power_kind`j' power_kind_raw    // temporarily rename variable for cleaning
	replace power_kind_raw = lower(power_kind_raw)   // normalize to lowercase
	merge m:1 power_kind_raw using "$interm\power_type_cleaned.dta"   // match against cleaned dictionary
	drop if _merge == 2        // drop unmatched records from using file
	cap drop _merge
	foreach i of numlist 1/11 {   // rename merged cleaned variables to reflect original suffix
		rename cleaned_unit_`i' power_cleaned_unit`j'_`i'
		rename cleaned_type_`i' power_cleaned_kind`j'_`i'
	}
	rename power_kind_raw power_kind`j'   // restore original variable name
}
keep file_name firm_number power_cleaned*   // keep only identifiers and cleaned power variables

* ------------------------------------------------------------
* STEP 3. Drop power_cleaned_kind* variables if all missing
* ------------------------------------------------------------
foreach var of varlist power_cleaned_kind* {
	capture assert mi(`var')   // check if variable is all missing
	if !_rc {
		drop `var'
		local unit_var = substr("`var'",19,.)   // extract suffix
		local unit_var = "power_cleaned_unit"+"`unit_var'"
		drop `unit_var'   // drop corresponding unit variable
	}
}

* ------------------------------------------------------------
* STEP 4. Reassign sequential suffixes to cleaned variables
* ------------------------------------------------------------
local i = 1
foreach var of varlist power_cleaned_unit* {
	rename `var' power_cleaned_unit`i'
	local i = `i'+1
}
local i = 1
foreach var of varlist power_cleaned_kind* {
	rename `var' power_cleaned_kind`i'
	local i = `i'+1
}

* ------------------------------------------------------------
* STEP 5. Reshape power variables to long form
* ------------------------------------------------------------
reshape long power_cleaned_unit power_cleaned_kind, i(file_name firm_number) j(iter)
drop if power_cleaned_kind == ""   // drop empty rows

* Merge with full cleaned reference file
merge m:1 power_cleaned_kind using `cleaned_strings'
drop if _merge == 2
gsort file_name firm_number iter
cap drop _merge iter
foreach var of varlist power_steam power_water power_wind power_horse power_hand {
	replace `var' = 0 if missing(`var')   // ensure missing dummies are set to zero
}

* Save as intermediate file
tempfile power_machine_1860
save `power_machine_1860', replace

* ------------------------------------------------------------
* STEP 6. Create establishment-level power source indicators
* ------------------------------------------------------------
use `power_machine_1860', clear
gen any_fire = strpos(power_cleaned_kind, "fire")>0   // temporary indicator for "fire" strings
keep file_name firm_number power_steam power_water power_wind power_horse power_hand any_fire
collapse (max) power_steam power_water power_wind power_horse power_hand any_fire, by(file_name firm_number)
tempfile power_1860
save `power_1860', replace

* ------------------------------------------------------------
* STEP 7. Extract machine information per establishment
* ------------------------------------------------------------
use `power_machine_1860', clear
keep file_name firm_number machine_dummy power_cleaned_unit power_cleaned_kind machine_category
keep if machine_dummy == 1
cap drop machine_dummy
bys file_name firm_number: gen iter = _n   // order machines by appearance
rename (power_cleaned_unit power_cleaned_kind machine_category) (machine_unit machine_kind machine_category)
reshape wide machine_unit machine_kind machine_category, i(file_name firm_number) j(iter)
tempfile machine_1860
save `machine_1860', replace

* ------------------------------------------------------------
* STEP 8. Merge cleaned power and machine data back to 1860 file
* ------------------------------------------------------------
use "${input}\1860_industry_assigned.dta", clear
merge 1:1 file_name firm_number using `power_1860', nogen
foreach var of varlist power_steam power_water power_wind power_horse power_hand {
	replace `var' = 0 if missing(`var')
}
merge 1:1 file_name firm_number using `machine_1860'
drop _merge

* ------------------------------------------------------------
* STEP 9. Industry-based corrections for mills
* ------------------------------------------------------------
replace power_steam = 1 if strpos(industry_raw,"steam") > 0 & inlist(ind_leontief, "lumber", "flour and grist mills")
replace power_water = 1 if strpos(industry_raw,"water") > 0 & inlist(ind_leontief, "lumber", "flour and grist mills")
replace power_steam = 1 if any_fire == 1 & inlist(ind_leontief, "lumber", "flour and grist mills")
drop any_fire 

* ------------------------------------------------------------
* STEP 10. Save final cleaned dataset
* ------------------------------------------------------------
save "$output\1860_power_machine_vars_cleaned.dta", replace