/* Purpose: Clean power_kind* and machine* in 1870 data. */ if "`1'"==""{ global CMF_pipeline "../.." } * Define folder globals for input, intermediate, and output global input "${CMF_pipeline}/5_industry_assignment/output" global interm "${CMF_pipeline}/6_power_machine_cleaning/intermediate_files" global output "${CMF_pipeline}/6_power_machine_cleaning/output" * ------------------------------------------------------------ * STEP 1. Import mapping between raw and cleaned power/machine strings * ------------------------------------------------------------ import excel "$interm\power_machine_category.xlsx", sheet("Sheet1") firstrow clear capture drop freq done // drop extra columns if they exist rename ctype power_cleaned_kind rename ctype_steam power_steam rename ctype_water power_water rename ctype_wind power_wind rename ctype_horse power_horse rename ctype_hand power_hand rename ctype_machine machine_dummy rename ctype_machine_category machine_category tempfile cleaned_strings save `cleaned_strings', replace // save mapping as temp file * ------------------------------------------------------------ * STEP 2. Clean power_kind* variables * ------------------------------------------------------------ use "${input}\1870_industry_assigned.dta", clear keep file_name firm_number power_kind* // keep identifiers and power_kind vars reshape long power_kind, i(file_name firm_number) j(iter) // reshape into long format drop if power_kind == "" // drop empty entries replace power_kind = lower(power_kind) // normalize to lowercase rename power_kind power_kind_raw recast str100 power_kind_raw, force // ensure consistent string length merge m:1 power_kind_raw using "$interm\power_type_cleaned.dta" // merge cleaned dictionary drop if _merge == 2 cap drop _merge cap drop iter power_kind_raw bys file_name firm_number: gen row_number = _n // index rows within establishments reshape long cleaned_unit_ cleaned_type_, i(file_name firm_number row_number) j(iter) drop if cleaned_type_ == "" // drop unmatched cap drop row_number rename (cleaned_unit_ cleaned_type_) (power_cleaned_unit power_cleaned_kind) tempfile power_kind_var_1870 save `power_kind_var_1870', replace * ------------------------------------------------------------ * STEP 3. Clean machine* variables * ------------------------------------------------------------ use "${input}\1870_industry_assigned.dta", clear keep file_name firm_number machine_description* machine_number* reshape long machine_description machine_number, i(file_name firm_number) j(iter) drop if machine_description == "" // drop empty rows destring machine_number, force replace // convert machine_number to numeric rename machine_description machine_description_raw replace machine_description_raw = lower(machine_description_raw) merge m:1 machine_description_raw using "$interm\machine_1870_cleaned.dta" // cleaned dictionary drop if _merge == 2 cap drop _merge replace machine_unit_1 = machine_number if !missing(machine_number) // use machine_number as unit if available keep file_name firm_number machine_unit* machine_description* cap drop machine_description_raw bys file_name firm_number: gen row_number = _n reshape long machine_unit_ machine_description_, i(file_name firm_number row_number) j(iter) drop if machine_description_ == "" cap drop row_number order file_name firm_number machine_unit_ machine_description_, first rename (machine_unit_ machine_description_) (power_cleaned_unit power_cleaned_kind) tempfile machine_var_1870 save `machine_var_1870', replace * ------------------------------------------------------------ * STEP 4. Concatenate cleaned power and machine data * ------------------------------------------------------------ use `power_kind_var_1870', clear append using `machine_var_1870' merge m:1 power_cleaned_kind using `cleaned_strings' drop if _merge == 2 gsort file_name firm_number iter cap drop _merge iter foreach var of varlist power_steam power_water power_wind power_horse power_hand { replace `var' = 0 if missing(`var') // replace missing dummies with zero } * Save as intermediate file tempfile power_machine_1870 save `power_machine_1870', replace * ------------------------------------------------------------ * STEP 5. Create establishment-level power source indicators * ------------------------------------------------------------ use `power_machine_1870', clear gen any_fire = strpos(power_cleaned_kind, "fire")>0 // detect "fire" in descriptions keep file_name firm_number power_steam power_water power_wind power_horse power_hand any_fire collapse (max) power_steam power_water power_wind power_horse power_hand any_fire, by(file_name firm_number) tempfile power_1870 save `power_1870', replace * ------------------------------------------------------------ * STEP 6. Extract machine information per establishment * ------------------------------------------------------------ use `power_machine_1870', clear keep file_name firm_number machine_dummy power_cleaned_unit power_cleaned_kind machine_category keep if machine_dummy == 1 cap drop machine_dummy bys file_name firm_number: gen iter = _n // order machines by appearance rename (power_cleaned_unit power_cleaned_kind machine_category) (machine_unit machine_kind machine_category) reshape wide machine_unit machine_kind machine_category, i(file_name firm_number) j(iter) tempfile machine_1870 save `machine_1870', replace * ------------------------------------------------------------ * STEP 7. Merge cleaned power and machine data back into 1870 file * ------------------------------------------------------------ use "${input}\1870_industry_assigned.dta", clear merge 1:1 file_name firm_number using `power_1870', nogen foreach var of varlist power_steam power_water power_wind power_horse power_hand { replace `var' = 0 if missing(`var') } merge 1:1 file_name firm_number using `machine_1870' drop _merge * ------------------------------------------------------------ * STEP 8. Industry-based corrections for mills * ------------------------------------------------------------ replace power_steam = 1 if strpos(industry_raw,"steam") > 0 & inlist(ind_leontief, "lumber", "flour and grist mills") replace power_water = 1 if strpos(industry_raw,"water") > 0 & inlist(ind_leontief, "lumber", "flour and grist mills") replace power_steam = 1 if any_fire == 1 & inlist(ind_leontief, "lumber", "flour and grist mills") drop any_fire * ------------------------------------------------------------ * STEP 9. Save final cleaned dataset * ------------------------------------------------------------ save "${output}\1870_power_machine_vars_cleaned.dta", replace