**********************************************************
**# Apply Early Stage Industry_raw -> industry cleaning #1
**********************************************************
*drop previously run variable as we adjust the codeflow will redo the process again.
*but we still keep the variable industry_raw for convenience.

// cap drop ind_broadest ind_leontief ind_granular ind_detailed 
// cap drop industry 
// cap drop industry_freq

*double check if industry_raw is trimmedbefore merge.
replace industry_raw = lower(industry_raw)
replace industry_raw  = trim(industry_raw)
replace industry_raw  = stritrim(industry_raw)
cap gen index = _n

tempfile main_data
save "`main_data'", replace
	
*merge the RA cleaned.
use "${strings}/product_name_appended_w_discs.dta", clear
drop product_name_intermediate correct_RA correct_RA_unsure comments unsure disc M F
ren product_name industry_raw
merge 1:m industry_raw using "`main_data'", keep(2 3) nogen

forval x = 1/4 {
	ren product_name`x' industry_clean`x'	
}
cap ren product_name5 industry_clean5
cap ren total_freq industry_freq
order industry_raw industry_clean* 
sort index
drop index
//combine industry 1, 2, 3, 4, 5
gen industry = ""
forval x = 1/4 {
	replace industry = industry + " // " + industry_clean`x' if industry_clean`x' != "" & industry != ""
	replace industry = industry + industry_clean`x' if industry_clean`x' != "" & industry == ""
}
cap replace industry = industry + " // " + industry_clean5 if industry_clean5 != ""
drop industry_clean*

preserve
	keep if industry != ""
	tempfile has_industry
	save "`has_industry'"
restore

keep if industry == ""
drop maker shop manufacture factory
replace industry = industry_raw

**For those industry_raw without industry, we do the minimum cleaning to mimic the format of "industry."
**i.e., change all & or , into //

*Get rid of extra spaces
replace industry = trim(industry)
replace industry = stritrim(industry)	//handy tip from Nick Cox

*Standardize "and"
replace industry = subinstr(industry, " and ", " & ", .)
replace industry = subinstr(industry, "&c", "& c", .)

*Fix super common misspellings
replace industry = subinstr(industry, "waggon", "wagon", .)
replace industry = subinstr(industry, "cabinett", "cabinet", .)

*Temporarily remove spaces for ease of use with regex
replace industry = subinstr(industry, " ", "_", .)

***Remove terms like "maker," "manufacture," etc. and create tags instead
*I think there are four basic categories: makers, shops, manufacturers, and factories

*Note that the following lists of words should basically go from long to short
*to avoid leaving stray r's or s's in

*Maker
gen maker = regexm(industry, "makers") | regexm(industry, "maker")
foreach str in makers maker making {
	replace maker = regexm(industry, "`str'") if maker == 0
	replace industry = subinstr(industry, "`str'", "", .)
}
//replace product_name_pp = subinstr(product_name, "maker", "", .) if maker == 1
//NOTE there's some bug that makes the line above not work. So I'm just going
//clean product_name for now and not bother with product_name_pp

*Shop
gen shop = regexm(industry, "shop")
foreach str in shops shop {
	replace shop = regexm(industry, "`str'") if shop == 0
	replace industry = subinstr(industry, "`str'", "", .)
}

*Manufacture
gen manufacture = regexm(industry, "manufacture")
foreach str in manufacturers_of_ manufacturer_of_ manufactory_of_ manufacture_of_ manufacturers manufacturer manufactures manufacture manufactory manufacturing manf{
	replace manufacture = regexm(industry, "`str'") if manufacture == 0
	replace industry = subinstr(industry, "`str'", "", .)
}
replace manufacture = regexm(industry, "ma?n?u?c?f[a-z]*$") if manufacture == 0
replace industry = ustrregexra(industry, "ma?n?u?c?f[a-z]*$","",.)


*Factory
gen factory = regexm(industry, "factory")
foreach str in factories_of factory_of factories factory {
	replace factory = regexm(industry, "`str'") if factory == 0
	replace industry = subinstr(industry, "`str'", "", .)
}

replace industry = subinstr(industry, "_", " ", .)
gen test = ustrregexra(industry, " ?& ?co?\.?$","")
replace industry = ustrregexra(industry, " ?& ?co?$","")
replace industry = ustrregexra(industry, " ?& \?$","")
replace industry = ustrregexra(industry, ",|and|&","//", .)
replace industry = ustrregexra(industry, "^ ?//","", .)

append using "`has_industry'"

replace industry = stritrim(industry)
replace industry = ustrtrim(industry)
replace industry = lower(industry)