/*-------------------------------------------------------------------------------
  safeappend.ado: a wrapper for -append- automatically resolving storage type
	mismatches between master and using datasets using -tostring-

    Copyright (C) 2016  Daniel Bela (daniel.bela@lifbi.de)

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-------------------------------------------------------------------------------*/
*! version 1.0 15 July - initial release
*! safeappend.ado: a wrapper for -append- automatically resolving storage type mismatches between master and using datasets using -tostring-
*! Daniel Bela (daniel.bela@lifbi.de), Leibniz-Institute for Educational Trajectories e.V. (LIfBi), Germany
program define safeappend , rclass
	// Stata version information: use unicode functions for string comparisons in Stata 14 and younger
	if (`c(stata_version)'>=14) {
		local substr_fct usubstr
		local regexm_fct ustrregexm
		local regexs_fct ustrregexs
		version 14
	}
	else {
		local substr_fct substr
		local regexm_fct regexm
		local regexs_fct regexs
		version 12
	}
	// declare macros
	local anything
	local plural
	local num
	local allparams
	local usingword
	local usingfile
	local usingfiles
	local usingfilecount
	local newparams
	local safeusing
	local masterlist
	local usinglist
	local bothlist
	local string_in_master
	local string_in_using
	local num_in_master
	local num_in_using
	local convert_in_master
	local convertcount_master 0
	local convert_in_using
	local convertcount_using 0
	local convertvar
	local testvar
	local keep
	local datasetname
	local restoresnapshot
	local filenum 0
	local filenumpos
	local returnmastername
	local returnusingname
	// parse parameters
	local allparams `macval(0)'
	syntax [anything(everything)] [ , GENerate(name) keep(string) * ]
	gettoken usingword usingfiles : anything
	if (`"`usingword'"'!="using") {
		display as error "using required"
		exit 100
	}
	// generate marker variable, if specified
	if (!missing(`"`generate'"')) {
		confirm new variable `generate'
		generate byte `generate'=`filenum'
		local filenumpos = _N + 1
	}
	// iterate over using files, check variable lists, and finally -append-
	local usingfilecount : word count `usingfiles'
	foreach usingfile of local usingfiles {
		local ++ filenum
		local masterlist
		local usinglist
		local bothlist
		local string_in_master
		local string_in_using
		local num_in_master
		local num_in_using
		local convert_in_master
		local convertcount_master 0
		local convert_in_using
		local convertcount_using`filenum' 0
		local safeusing`filenum' `usingfile'
		// compare file variable lists
		unab masterlist : _all
		quietly : describe using `"`usingfile'"' , varlist
		local usinglist `r(varlist)'
		local bothlist : list masterlist & usinglist
		if (!missing(`"`keep'"')) local bothlist : list bothlist & keep
		// if matching variables are present in both files: check their storage types
		if (!missing(`"`bothlist'"')) {
			// fill lists in master and using
			foreach dataset in master using {
				if (`"`dataset'"'=="using") {
					// save data snapshot, switch to using dataset
					quietly : snapshot save , label("masterfile for safeappend.ado")
					local restoresnapshot `r(snapshot)'
					quietly : use `bothlist' using `"`usingfile'"' in 1 , clear
				}
				foreach testvar of local bothlist {
					if (`substr_fct'(`"`: type `testvar''"',1,3)=="str") local string_in_`dataset' : list string_in_`dataset' | testvar
					else local num_in_`dataset' : list num_in_`dataset' | testvar
				}
			}
			// compare lists
			local convert_in_master : list num_in_master & string_in_using
			local convert_in_using : list num_in_using & string_in_master
			// convert everything that needs to be converted in using and master
			foreach dataset in using master {
				if (!missing(`"`convert_in_`dataset''"')) {
					if (`"`dataset'"'=="using") {
						if (missing(`"`keep'"')) quietly : use `"`usingfile'"' , clear
						else quietly : use `keep' using `"`usingfile'"' , clear
						local datasetname using`filenum'
						local datasettag using file `filenum'
					}
					else {
						if (`filenum'==1) {
							local datasetname master
							local datasettag master file
						}
						else {
							local datasetname="using"+strofreal(`filenum'-1)
							local datasettag="all files up to using file "+strofreal(`filenum'-1)
						}
					}
					unstring `convert_in_`dataset'' , tag(`datasettag')
					local convertcount_`datasetname': word count `convert_in_`dataset''
					if (`"`dataset'"'=="using") {
						// temporarily save
						tempfile safeusing`filenum'
						quietly : save `"`safeusing`filenum''"' , replace
						// return results
						return scalar convertcount_`datasetname'=`convertcount_`datasetname''
						if (`convertcount_`datasetname''>0) {
							if (`convertcount_`datasetname''>1) local plural s
							else local plural
							display as text in smcl `" Note: {cmd:safeappend} force-converted {it:`convertcount_`datasetname''} variable`plural' in `datasettag' dataset to string: {it:`convert_in_`dataset''}"'
							return local convertlist_`datasetname' `convert_in_`dataset''
						}
					}
					else {
						// return results
						if (`filenum'!=1) local upto upto
						else local upto
						return scalar convertcount_`upto'`datasetname'=`convertcount_`datasetname''
						if (`convertcount_`datasetname''>0) {
							if (`convertcount_`datasetname''>1) local plural s
							else local plural
							display as text in smcl `" Note: {cmd:safeappend} force-converted {it:`convertcount_`datasetname''} variable`plural' in `datasettag' dataset to string: {it:`convert_in_`dataset''}"'
							return local convertlist_`upto'`datasetname' `convert_in_`dataset''
						}
					}
				}
				// jump back to master
				if (`"`dataset'"'=="using") {
					snapshot restore `restoresnapshot'
					snapshot erase `restoresnapshot'
				}
			}
		}
		// append with safe-to-append dataset
		capture noisily _append using `"`safeusing`filenum''"', keep(`keep') `options'
		if (_rc!=0) {
			if (!missing(`"`generate'"')) {
				quietly drop if `generate' > 0
				drop `generate'
			}
			exit _rc
		}
		if (!missing(`"`generate'"')) {
			quietly replace `generate'=`filenum' in `filenumpos'/L
			local filenumpos=_N+1
		}
	}
	// return result: number of using files
	return scalar usingcount=`filenum'
	// exit
	exit 0
end
// sub-program executing -tostring- or -destring- to a list of variables
program unstring
	syntax varlist(min=1) , tag(string)
	local var
	tempvar marker
	foreach var of local varlist {
		quietly {
			generate `marker'=(missing(`var'))
			tostring `var' , replace force usedisplayformat
			replace `var'="" if (`marker')
			drop `marker'
		}
		char define `var'[safeappend] force-converted to string in `tag' by safeappend.ado
		char define `var'[tostring]
	}
	exit 0
end
// EOF