/*-------------------------------------------------------------------------------
  casi_popro.ado: Stata post-processor for NEPS CASI raw data

    Copyright (C) 2023-  LIfBi | Survey Technology (surveytechnologie@lifbi.de)
                            Daniel Bela

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <https://www.gnu.org/licenses/>.

-------------------------------------------------------------------------------*/
*! casi_popro.ado: Stata post-processor for NEPS CASI raw data
*! Daniel Bela (surveytechnologie@lifbi.de), Leibniz Institute for Educational Trajectories (LIfBi), Germany
*! version 2.6.0 09 Mai 2025 - hotfix adding exit survey data to dataset; prettify LimeSurvey technical variables
*! version 2.5.0 13 December 2024 - added feature to erase all observations black listed in the id list (by indicator variable 'kill_switch')
*! version 2.4.0 05 November 2024 - added custom data processing for network module survey
*! version 2.3.3 05 November 2024 - (hopefully) final bugfix for working around A104 TBT files not in subdirectories
*! version 2.3.2 02 September 2024 - yet another bugfix for working around A104 TBT files not in subdirectories
*! version 2.3.1 12 August 2024 - bugfix for not aborting file copy procedure in case of no exit survey data present
*! version 2.3.0 29 July 2024 - reworked handling exit survey data, avoid exporting directories containing para data without survey data
*! version 2.2.3 29 July 2024 - bugfix for working around A104 TBT files not in subdirectories
*! version 2.2.2 13 May 2024 - new directory structure also for LS modules; a little more chatty error messages
*! version 2.2.1 12 April 2024 - minor bugfixes for CAWI mode
*! version 2.2.0 08 April 2024 - reworked directory structure for output
*! version 2.1.1 27 March 2024 - add '*.sql' files from TBT root directory to TBT export data
*! version 2.1.0 20 March 2024 - add overview dataset with testgroup/id-mappings
*! version 2.0.10 04 March 2024 - explicit subdirectory for ZEUS logging archives
*! version 2.0.9 26 February 2024 - allow focus person type C
*! version 2.0.8 08 February 2024 - auto-rename merged id variable based on target focus person identifier
*! version 2.0.7 09 January 2024 - make temporary extract paths optional arguments
*! version 2.0.6 09 January 2024 - report 'lastpage' for dropped, empty observations, if possible; make dropping empty observations an option
*! version 2.0.5 18 December 2023 - tolerate leading/trailing spaces around tokens when handling Matomo tokens
*! version 2.0.4 30 November 2023 - bugfix: no redundant type conversion before appending datasets
*! version 2.0.3 14 November 2023 - added robustness against empty Matomo or exit survey datasets
*! version 2.0.2 06 October 2023 - allow focus person type I
*! version 2.0.1 03 August 2023 - reworked detection of episodical variables (and datasets)
*! version 2.0.0 31 July 2023 - implemented handling if instrument-wise Matomo exports, including auto-fixing Matomo tokens
*! version 1.9.0 29 July 2023 - correctly process exit survey data (paradata from force-canceling or timeout events)
*! version 1.8.0 14 July 2023 - updated routine for changed file structure in A105, introduced logic to auto-separate address data from regular data
*! version 1.7.1 12 Mai 2023 - bugfix handling of auto-dropped observations
*! version 1.7.0 19 April 2023 - don't try to export / process datasets without observations; include the ZIPped raw data in output directory
*! version 1.6.4 19 April 2023 - bugfix in handling string list of dropped ids
*! version 1.6.3 02 March 2023 - added failsafe mechanism to make sure observations dropped due to lack of valid data in one data source can safely be dropped from another, even if dropping in this other datasource would be skipped due to no observations left
*! version 1.6.2 28 February 2023 - add '*.db' files from TBT root directory to TBT export data
*! version 1.6.1 28 February 2023 - do not drop observations without valid information in case the result would be an empty dataset file
*! version 1.6 24 February 2023 - added explicit routine to export LimeSurvey timing data
*! version 1.5 31 January 2023 - added CAWI wrapper -cawi_popro-
*! version 1.4 20 December 2022 - added sub-option -nofakeids- to -id_list-, and default mechanism to create fake ids; remove requirement for target id to be unique in -id_list-; be more robust when bulk erasing files (wait 1s and retry if a single rm call fails)
*! version 1.3 19 December 2022 - added routine to raw-copy TBT content data recursively
*! version 1.2 16 December 2022 - bugfix update: tolerate $ signs in file or directory names, tolerate non-unique tokens in source data
*! version 1.1 07 December 2022 - reduced version requirement to Stata 16.1
*! version 1.0 23 November 2022 - added usage message (full help file will follow)
*! version 0.6 22 November 2022 - added file name convention mechanics
*! version 0.5 22 November 2022 - added inserting export host name and time stamp, and option to omit it
*! version 0.4 22 November 2022 - added option to insert version number into file names
*! version 0.3 21 November 2022 - added detection and extraction of episodical data
*! version 0.2 16 November 2022 - added automatic id change integration
*! version 0.1 02 November 2022 - initial development release
program define casi_popro, nclass
    version 16.1

    **# define locals required for the program
    tempname unpackdir_stub
    local limesurvey_subdirectory limesurvey
    local survey_subdir_stub instrument_
    local matomo_subdirectory matomo
    local zeus_subdirectory zeus_tbt
    local prepared_id_files matomo_events matomo_visitors testgrouplist
    local episode_module_count 0
    local tbt_content_found 0
    local tbt_blacklisted_dirs \$Erhebungsdarstellung _Erhebungsdarstellung ZEUS_Logger
    local arrow=ustrunescape("\u2794")
    local matomo_data_detected 0
    local exit_survey_required startdate canceled timeout originsid
    local kill_switch_var kill_switch
    tempname frm_importfile frm_exitimportfile frm_exitdata frm_matomo_events frm_matomo_visitors frm_reshape_workspace frm_idselector frm_testgrouplist frm_network_survey_handling
    frame create `frm_importfile'
    frame create `frm_exitimportfile'
    frame create `frm_network_survey_handling'
    frame create `frm_matomo_events'
    frame create `frm_matomo_visitors'
    frame create `frm_reshape_workspace'
    frame create `frm_idselector'
    tempfile append_file
    tempvar row_nonmissings

    **# if help message requested, or no options given, drop usage notice
    if ( missing(`"`macval(0)'"') | inlist(`"`macval(0)'"',"--help","help","/help","help","?","--?","/?") ) {
        _usage_msg
        exit 0
    }

    **# parse user syntax
    capture : noisily syntax ,                  ///
        studynumber(name)                       /// the NEPS (sub) study number the data originates from
        [                                       ///
        inputdirectory(string)                  /// name of the directory with input (ZIP) files
        outputdirectory(string)                 /// name of the output directory
        temp_unpack_path(string)                /// directory to temporarily extract files to
        temp_address_unpack_path(string)        /// directory to temporarily extract address files to
        id_list(string)                         /// file name with a correspondence list of ids to insert into the result data;
                                                ///     with optional 'keepsource' internal sub-option to not erase the original variable,
                                                ///     and 'replace' sub-option to replace the target variable in the data, if already present,
                                                ///     and 'nofakeids' sub-option to prevent creating fake-ids for observations without ids in id list
        version(string)                         /// a version string to append to the result dataset file names
        focusperson(string)                     /// the letter denoting the focus person of the survey, most likely "T"
        noexporttimestamp                       /// do not include variables with the export host name and time stamp in the datasets
        nocleanup                               /// do not clean up temporary extract folder when finishing
        address_surveys(numlist integer sort)   /// list of LimeSurvey SIDs to be treated as address data
        exit_surveys(numlist integer sort)      /// list of LimeSurvey SIDs to be treated exit surveys
        network_surveys(numlist integer sort)   /// list of LimeSurvey SIDs to be treated network module surveys
        network_select_value(integer 1)         /// value to count as "selected" in network module surveys
        network_unselected_value(integer 0)     /// value to count as "unselected" in network module surveys
        network_varlist_clusters(string)        /// quoted clusters of variable lists to process in network module surveys
        id_variable(namelist)                   /// name of the central if variable (first varname will be used in export, all other varnames will be renamed to it if present)
        test_observation_regex(string)          /// regular expression matching test ids / tokens that shall be removed before export
        dropvars(string)                        /// variable list to drop from LimeSurvey exports
        nodropofemptycases                      /// do not drop empty observations (i.e. observations without valid data in any variable) from datasets
        ignore_duplicate_vars(namelist)         /// variable list to ignore when checking for auto-resolvable duplicate observations
        ignore_duplicate_varlabelregex(string)  /// regular expression for variable labels to ignore when checking for auto-resolvable duplicate observations
        replace                                 /// replace output files, if already present
        spellvarname(string)                    /// the variable name to use as a spell index when creating episodical datasets
        reshape_threshold(integer 2)            /// number of loop stubs to trigger classifying a module as episodical
        keep_empty_spells                       /// specifies that spells without content in the spell variables are to be kept, instead of deleted, after reshape
        spell_unused_value(integer -99)         /// the value indicating that a wide variable is unused in a spell loop
        filtered_missing(numlist integer)       /// the value(s) indicating that a numeric variable has been filtered
        filtered_stringmissing(string asis)     /// the value(s) indicating that a string variable has been filtered
        join_threshold(integer 2)               /// number of common variables required to join two episodical modules into one
        cawimode                                /// only do the processing parts applicable to CAWI exports, no unZIPping and such
        debug                                   /// print debug output
        ]
    if ( _rc!=0 ) {
        _usage_msg
        exit `c(rc)'
    }
    local mode_name=cond(!missing(`"`cawimode'"'),"cawi","casi")
    local upper_mode_name=ustrupper(`"`mode_name'"')

    **# report version information before doing anything else
    noisily: which casi_popro

    **# set global debug option
    global casi_popro_debug = ( !missing(`"`debug'"') )

    **# check option `inputdirectory'
    if ( missing(`"`inputdirectory'"') ) {
        _set_default_option inputdirectory , default(./input)
    }

    **# check user option `outputdirectory'
    if ( missing(`"`outputdirectory'"') ) {
        _set_default_option outputdirectory , default(./output)
    }
    _create_maybeexistent_dir `outputdirectory'

    **# check user option `temp_unpack_path'
    if ( missing(`"`temp_unpack_path'"') ) {
        _set_default_option temp_unpack_path , default(`c(tmpdir)'`=cond(inlist(usubstr(`"`c(tmpdir)'"',-1,1),`"/"',`"\"'),"","/")'`unpackdir_stub'_casi_popro)
    }
    _create_maybeexistent_dir `temp_unpack_path', replace

    **# check user option `temp_unpack_path'
    if ( missing(`"`temp_address_unpack_path'"') ) {
        _set_default_option temp_address_unpack_path , default(`temp_unpack_path'_addressdata)
    }
    _create_maybeexistent_dir `temp_address_unpack_path', replace

    **# check option `id_variable'
    if ( missing(`"`id_variable'"') ) {
        _set_default_option id_variable , default(token index)
    }
    local main_idvarname : word 1 of `id_variable'
    local other_idvarnames : list id_variable - main_idvarname

    **# check option `test_observation_regex'
    if ( missing(`"`test_observation_regex'"') ) {
        _set_default_option test_observation_regex , default(^test[0-9+](\w+)?$)
    }

    **# check option `dropvars'
    if ( missing(`"`dropvars'"') ) {
        _set_default_option dropvars , default(id seed startlanguage OUTnavigate* duration_navigate*)
    }

    **# check option `ignore_duplicate_vars'
    if ( missing(`"`ignore_duplicate_vars'"') ) {
        _set_default_option ignore_duplicate_vars , default(`dropvars' submitdate lastpage startdate datestamp casi_export_host casi_export_time)
    }

    **# check option `ignore_duplicate_varlabelregex'
    if ( missing(`"`ignore_duplicate_varlabelregex'"') ) {
        _set_default_option ignore_duplicate_varlabelregex , default(.*\[(HILF)|(AUTO)\].*)
    }

    **# check option `filtered_missing'
    if ( missing(`"`filtered_missing'"') ) {
        _set_default_option filtered_missing, default(-99)
    }

    **# check option `filtered_stringmissing'
    if ( missing(`"`filtered_stringmissing'"') ) {
        _set_default_option filtered_stringmissing, default(`"-99 (filterbedingt fehlend)"',`"-99 (filtered)"',`"-99"')
    }

    **# check option `string_replacements'
    if ( missing(`"`string_replacements'"') ) {
        _set_default_option string_replacements, default(`"-99"'|`"-99 (filtered)"' `"-99 (filterbedingt fehlend)"'|`"-99 (filtered)"') // TODO: alle strmissings einpflegen
    }

    **# check option `spellvarname'
    if ( missing(`"`spellvarname'"') ) {
        _set_default_option spellvarname , default(spell)
    }

    **# check option `address_surveys'
    if ( missing(`"`address_surveys'"') ) {
        _set_default_option address_surveys , default(752494)
    }

    **# check option `exit_surveys'
    if ( missing(`"`exit_surveys'"') ) {
        _set_default_option exit_surveys , default(613757)
    }

    **# check option `network_surveys'
    if ( missing(`"`network_surveys'"') ) {
        _set_default_option network_surveys , default(967002 1102000)
    }

    **# check option `network_varlist_clusters'
    if ( missing(`"`network_varlist_clusters'"') ) {
        _set_default_option network_varlist_clusters , default(`""t331001 t331002 t331003 t331004 t331005 t331006 t331007 t331008 t331009 t331010 t331011 t331012 t331013 t331014 t331015 t331016 t331017 t33101 t331019 t331020 t331021 t331022 t331023 t331024 t331025 t331026 t331027 t331028 t331029 t331030 t331031 t331032 t331033 t331034 t331035" "t331038 t331039 t331040 t331041 t331042 t331043 t331044 t331045 t331046 t331047 t331048 t331049 t331050 t331051 t331052 t331053 t331054 t331055 t331056 t331057 t331058 t331059 t331060 t331061 t331062 t331063 t331064 t331065 t331066 t331067 t331068 t331069 t331070 t331071 t331072" "t331075 t331076 t331077 t331078 t331079 t331080 t331081 t331082 t331083 t331084 t331085 t331086 t331087 t331088 t331089 t331090 t331091 t331092 t331093 t331094 t331095 t331096 t331097 t331098 t331099 t331100 t331101 t331102 t331103 t331104 t331105 t331106 t331107 t331108 t331109" "t331112 t331113 t331114 t331115 t331116 t331117 t331118 t331119 t331120 t331121 t331122 t331123 t331124 t331125 t331126 t331127 t331128 t331129 t331130 t331131 t331132 t331133 t331134 t331135 t331136 t331137 t331138 t331139 t331140 t331141 t331142 t331143 t331144 t331145 t331146" "t331149 t331150 t331151 t331152 t331153 t331154 t331155 t331156 t331157 t331158 t331159 t331160 t331161 t331162 t331163 t331164 t331165 t331166 t331167 t331168 t331169 t331170 t331171 t331172 t331173 t331174 t331175 t331176 t331177 t331178 t331179 t331180 t331181 t331182 t331183" "t331186 t331187 t331188 t331189 t331190 t331191 t331192 t331193 t331194 t331195 t331196 t331197 t331198 t331199 t331200 t331201 t331202 t331203 t331204 t331205 t331206 t331207 t331208 t332296 t331209 t331210 t331211 t331212 t331213 t331214 t331215 t331216 t331217 t331218 t331219" "t331222 t331223 t331224 t331225 t331226 t331227 t331228 t331229 t331230 t331231 t331232 t331233 t331234 t331235 t331236 t331237 t331238 t331239 t331240 t331241 t331242 t331243 t331244 t331245 t331246 t331247 t331248 t331249 t331250 t331251 t331252 t331253 t331254 t331255 t331256" "t331259 t331260 t331261 t331262 t331263 t331264 t331265 t331266 t331267 t331268 t331269 t331270 t331271 t331272 t331273 t331274 t331275 t331276 t331277 t331278 t331279 t331280 t331281 t331282 t331283 t331284 t331285 t331286 t331287 t331288 t331289 t331290 t331291 t331292 t331293 t331294 t331295 t331296 t331297 t331298 t331299 t331300 t331301 t331302 t331303 t331304 t331305 t331306 t331307 t331308 t331309 t331310 t331311 t331312 t331313 t331314 t331315 t331316 t331317 t331318 t331319 t331320 t331321 t331322 t331323 t331324 t331325 t331326 t331327 t331328 t331329 t331330 t331331 t331332 t331333 t331334 t331335 t331336 t331337 t331338 t331339 t331340 t331341 t331342 t331343 t331344 t331345 t331346 t331347 t331348 t331349 t331350 t331351 t331352 t331353 t331354 t331355 t331356 t331357 t331358 t331359 t331360 t331361 t331362 t331363 t331364 t331365 t331366 t331367 t331368 t331369 t331370 t331371 t331372 t331373 t331374 t331375 t331376 t331377 t331378 t331379 t331380 t331381 t331382 t331383 t331384 t331385 t331386 t331387 t331388 t331389 t331390 t331391 t331392 t331393 t331394 t331395 t331396 t331397 t331398 t331399 t331400 t331401 t331402 t331403 t331404 t331405 t331406 t331407 t331408 t331409 t331410 t331411 t331412 t331413 t331414 t331415 t331416 t331417 t331418 t331419 t331420 t331421 t331422 t331423 t331424 t331425 t331426 t331427 t331428 t331429 t331430 t331431 t331432 t331433 t331434 t331435 t331436 t331437 t331438 t331439 t331440 t331441 t331442 t331443 t331444 t331445 t331446 t331447 t331448 t331449 t331450 t331451 t331452 t331453 t331454 t331455 t331456 t331457 t331458 t331459 t331460 t331461 t331462 t331463 t331464 t331465 t331466 t331467 t331468 t331469 t331470 t331471 t331472 t331473 t331474 t331475 t331476 t331477 t331478 t331479 t331480 t331481 t331482 t331483 t331484 t331485 t331486 t331487 t331488 t331489 t331490 t331491 t331492 t331493 t331494 t331495 t331496 t331497 t331498 t331499 t331500 t331501 t331502 t331503" "t331506 t331507 t331508 t331509 t331510 t331511 t331512 t331513 t331514 t331515 t331516 t331517 t331518 t331519 t331520 t331521 t331522 t331523 t331524 t331525 t331526 t331527 t331528 t331529 t331530 t331531 t331532 t331533 t331534 t331535 t331536 t331537 t331538 t331539 t331540" "t331544 t331545 t331546 t331547 t331548 t331549 t331550 t331551 t331552 t331553 t331554 t331555 t331556 t331557 t331558 t331559 t331560 t331561 t331562 t331563 t331564 t331565 t331566 t331567 t331568 t331569 t331570 t331571 t331572 t331573 t331574 t331575 t331576 t331577 t331578" "t331582 t331583 t331584 t331585 t331586 t331587 t331588 t331589 t331590 t331591 t331592 t331593 t331594 t331595 t331596 t331597 t331598 t331599 t331600 t331601 t331602 t331603 t331604 t331605 t331606 t331607 t331608 t331609 t331610 t331611 t331612 t331613 t331614 t331615 t331616" "t331620 t331621 t331622 t331623 t331624 t331625 t331626 t331627 t331628 t331629 t331630 t331631 t331632 t331633 t331634 t331635 t331636 t331637 t331638 t331639 t331640 t331641 t331642 t331643 t331644 t331645 t331646 t331647 t331648 t331649 t331650 t331651 t331652 t331653 t331654" "t331658 t331659 t331660 t331661 t331662 t331663 t331664 t331665 t331666 t331667 t331668 t331669 t331670 t331671 t331672 t331673 t331674 t331675 t331676 t331677 t331678 t331679 t331680 t331681 t331682 t331683 t331684 t331685 t331686 t331687 t331688 t331689 t331690 t331691 t331692" "t331696 t331697 t331698 t331699 t331700 t331701 t331702 t331703 t331704 t331705 t331706 t331707 t331708 t331709 t331710 t331711 t331712 t331713 t331714 t331715 t331716 t331717 t331718 t331719 t331720 t331721 t331722 t331723 t331724 t331725 t331726 t331727 t331728 t331729 t331730" "t331734 t331735 t331736 t331737 t331738 t331739 t331740 t331741 t331742 t331743 t331744 t331745 t331746 t331747 t331748 t331749 t331750 t331751 t331752 t331753 t331754 t331755 t331756 t331757 t331758 t331759 t331760 t331761 t331762 t331763 t331764 t331765 t331766 t331767 t331768" "t331772 t331773 t331774 t331775 t331776 t331777 t331778 t331779 t331780 t331781 t331782 t331783 t331784 t331785 t331786 t331787 t331788 t331789 t331790 t331791 t331792 t331793 t331794 t331795 t331796 t331797 t331798 t331799 t331800 t331801 t331802 t331803 t331804 t331805 t331806" "t331810 t331811 t331812 t331813 t331814 t331815 t331816 t331817 t331818 t331819 t331820 t331821 t331822 t331823 t331824 t331825 t331826 t331827 t331828 t331829 t331830 t331831 t331832 t331833 t331834 t331835 t331836 t331837 t331838 t331839 t331840 t331841 t331842 t331843 t331844" "t331848 t331849 t331850 t331851 t331852 t331853 t331854 t331855 t331856 t331857 t331858 t331859 t331860 t331861 t331862 t331863 t331864 t331865 t331866 t331867 t331868 t331869 t331870 t331871 t331872 t331873 t331874 t331875 t331876 t331877 t331878 t331879 t331880 t331881 t331882" "t331886 t331887 t331888 t331889 t331890 t331891 t331892 t331893 t331894 t331895 t331896 t331897 t331898 t331899 t331900 t331901 t331902 t331903 t331904 t331905 t331906 t331907 t331908 t331909 t331910 t331911 t331912 t331913 t331914 t331915 t331916 t331917 t331918 t331919 t331920" "t331924 t331925 t331926 t331927 t331928 t331929 t331930 t331931 t331932 t331933 t331934 t331935 t331936 t331937 t331938 t331939 t331940 t331941 t331942 t331943 t331944 t331945 t331946 t331947 t331948 t331949 t331950 t331951 t331952 t331953 t331954 t331955 t331956 t331957 t331958" "t331962 t331963 t331964 t331965 t331966 t331967 t331968 t331969 t331970 t331971 t331972 t331973 t331974 t331975 t331976 t331977 t331978 t331979 t331980 t331981 t331982 t331983 t331984 t331985 t331986 t331987 t331988 t331989 t331990 t331991 t331992 t331993 t331994 t331995 t331996" "t332001 t332002 t332003 t332004 t332005 t332006 t332007 t332008 t332009 t332010 t332011 t332012 t332013 t332014 t332015 t332016 t332017 t332018 t332019 t332020 t332021 t332022 t332023 t332024 t332025 t332026 t332027 t332028 t332029 t332030 t332031 t332032 t332033 t332034 t332035" "t332039 t332040 t332041 t332042 t332043 t332044 t332045 t332046 t332047 t332048 t332049 t332050 t332051 t332052 t332053 t332054 t332055 t332056 t332057 t332058 t332059 t332060 t332061 t332062 t332063 t332064 t332065 t332066 t332067 t332068 t332069 t332070 t332071 t332072 t332073" "t332076 t332077 t332078 t332079 t332080 t332081 t332082 t332083 t332084 t332085 t332086 t332087 t332088 t332089 t332090 t332091 t332092 t332093 t332094 t332095 t332096 t332097 t332098 t332099 t332100 t332101 t332102 t332103 t332104 t332105 t332106 t332107 t332108 t332109 t332110" "t332113 t332114 t332115 t332116 t332117 t332118 t332119 t332120 t332121 t332122 t332123 t332124 t332125 t332126 t332127 t332128 t332129 t332130 t332131 t332132 t332133 t332134 t332135 t332136 t332137 t332138 t332139 t332140 t332141 t332142 t332143 t332144 t332145 t332146 t332147" "t332150 t332151 t332152 t332153 t332154 t332155 t332156 t332157 t332158 t332159 t332160 t332161 t332162 t332163 t332164 t332165 t332166 t332167 t332168 t332169 t332170 t332171 t332172 t332173 t332174 t332175 t332176 t332177 t332178 t332179 t332180 t332181 t332182 t332183 t332184" "t332187 t332188 t332189 t332190 t332191 t332192 t332193 t332194 t332195 t332196 t332197 t332198 t332199 t332200 t332201 t332202 t332203 t332204 t332205 t332206 t332207 t332208 t332209 t332210 t332211 t332212 t332213 t332214 t332215 t332216 t332217 t332218 t332219 t332220 t332221" "t332224 t332225 t332226 t332227 t332228 t332229 t332230 t332231 t332232 t332233 t332234 t332235 t332236 t332237 t332238 t332239 t332240 t332241 t332242 t332243 t332244 t332245 t332246 t332247 t332248 t332249 t332250 t332251 t332252 t332253 t332254 t332255 t332256 t332257 t332258" "t332261 t332262 t332263 t332264 t332265 t332266 t332267 t332268 t332269 t332270 t332271 t332272 t332273 t332274 t332275 t332276 t332277 t332278 t332279 t332280 t332281 t332282 t332283 t332284 t332285 t332286 t332287 t332288 t332289 t332290 t332291 t332292 t332293 t332294 t332295""')
    }

    **# check option `focusperson'
    if ( missing(`"`focusperson'"') ) {
        _set_default_option focusperson , default(T)
    }
    local focusperson = ustrupper(`"`focusperson'"')
    if ( !inlist(`"`focusperson'"', "T", "P", "H", "E", "I", "C") ) {
        _print_error `"Invalid specification for {opt focusperson()}; valid values are {input:T} for target person, {input:P} for parent, {input:H} for headmaster, {input:E} for educator, {input:I} for interviewer, or {input:C} for coordinator."'
        _usage_msg
        exit 198
    }

    **# check (and open) id_list, if specified
    if ( !missing(`"`id_list'"') ) {
        _parse_id_list_opt `id_list'
        confirm file `id_list_path'
        _print_debug … consistency check passed: source file {input:`id_list_path'} is present
        mata : st_local("id_list_ext",pathsuffix(st_local("id_list_path")))
        if ( inlist(`"`id_list_ext'"',".dta","") ) {
            local open_cmd "use"
            local open_opts
        }
        else if ( inlist(`"`id_list_ext'"',".xlsx",".xls") ) {
            local open_cmd "import excel"
            local open_opts ", firstrow"
        }
        else if ( inlist(`"`id_list_ext'"',".csv",".tsv") ) {
            local open_cmd "import delimited"
            local open_opts ", varnames(1) case(preserve)"
        }
        else {
            _print_error File extension {input:`id_list_ext'} not supported for reading id list; please correct {opt id_list} option
            _usage_msg
            exit 198
        }
        _print_debug … consistency check passed: file extension {input:`id_list_ext'} is supported
        tempname frm_id_list
        frame create `frm_id_list'
        _print_info Ingesting ID list from {input:`id_list_ext'} file…
        frame `frm_id_list' {
            quietly : `open_cmd' `id_list_path' `open_opts'
            capture : confirm variable `kill_switch_var' , exact
            if ( _rc != 0 ) {
                _print_warning … black list variable {input:`kill_switch_var'} not found in id list
                generate `kill_switch_var'=0
            }
            else {
                _print_debug … consistency check passed: black list variable {input:`kill_switch_var'} is present
            }
            capture : confirm variable `main_idvarname' , exact
            if ( _rc != 0 ) {
                _print_error ID variable {input:`main_idvarname'} not found in id list
                exit 459
            }
            _print_debug … consistency check passed: main ID variable {input:`main_idvarname'} is present
            capture : isid `main_idvarname'
            if ( _rc != 0 ) {
                _print_error ID variable {input:`main_idvarname'} not uniquely identifying observations in id list
                exit 459
            }
            _print_debug … consistency check passed: main ID variable {input:`main_idvarname'} is uniquely identifying observations
            unab id_list_vars : _all
            local id_list_othervars : list id_list_vars - main_idvarname
            local idlist_idvarname : word 1 of `id_list_othervars'
            if (`"`focusperson'"' == "T") {
                local target_idvarname idtarget
            }
            else if (`"`focusperson'"' == "P") {
                local target_idvarname idparent
            }
            else if (`"`focusperson'"' == "H") {
                local target_idvarname idheadmaster
            }
            else if (`"`focusperson'"' == "E") {
                local target_idvarname idteach
            }
            else if (`"`focusperson'"' == "I") {
                local target_idvarname idinterviewer
            }
            else if (`"`focusperson'"' == "C") {
                local target_idvarname idcoordinator
            }
            else {
                /* not reached */
                error 459
            }
            local reportaddon
            if ( `"`idlist_idvarname'"' != `"`idtarget_idvarname'"' ) {
                rename `idlist_idvarname' `target_idvarname'
                local reportaddon `" (renamed from id list variable {input:`idlist_idvarname'})"'
            }
            format `target_idvarname' %30.0g
            _print_info … target ID variable will be {input:`target_idvarname'}`reportaddon'
        }
    }

    **# check prerequisites regarding ADO packages
    _check_ado_presence filesearch

    **# CAWI mode: save 'fake' ZIP-path, continue without extracting anything
    if ( !missing(`"`cawimode'"') ) {
        local maindir : dir "`inputdirectory'" dirs "*" , respectcase
        local maindircount : word count `maindir'
        if ( `maindircount' != 1 ) {
            _print_error expecting exactly {input:1} subdirectory to {input:inputdirectory(`inputdirectory')}, holding export data; but found {input:`=cond(`maindircount'==0,"none","`maindircount'")'}
            if ( `maindircount' != 0 ) {
                _print_error (namely: {input:`maindir'})
            }
            exit 459
        }
        local zipcounter 1
        local filecounter 1
        local unpack_dir `temp_unpack_path'/`filecounter'
        capture : _export_name_check `maindir'
        local zipfile_`filecounter'_maindir `r(dirname)'
        if ( _rc == 0 ) {
            _print_debug export {result:`filecounter'} is from host {result:`export_host'}, performed at {result:`export_time_for_humans'}
            local zipfile_`filecounter'_hostname `export_host'
            local zipfile_`filecounter'_export_time `export_time'
            local zipfile_`filecounter'_readable_time `export_time_for_humans'
            _create_maybeexistent_dir `unpack_dir'
            _create_maybeexistent_dir `unpack_dir'/`zipfile_`filecounter'_maindir'
            _recurse_and_copy_dir, from(`inputdirectory'/`zipfile_`filecounter'_maindir') to(`unpack_dir'/`zipfile_`filecounter'_maindir'/`limesurvey_subdirectory')
        }
        else {
            _print_warning Top level directory name in export {result:`filecounter'} does not seem right: {input:`zipfile_`filecounter'_maindir'}
        }
    }

    **# regular mode: extract ZIP archives and save information
    else {
        **## prepare test group tracking frame
        frame create `frm_testgrouplist' str20(token) double(tbt_idtarget) str30(casi_export_host) double(casi_export_time) byte(contains_) str6(survey_type)
        frame `frm_testgrouplist' {
            _insert_export_variables, mode(casi) nogenerate
            label define yesno 0 "no" 1 "yes"
            label values contains_ yesno
            format tbt_idtarget %30.0g
        }

        **## gather ZIP files
        filesearch *.zip , directory(`inputdirectory') fullpath recursive local(zipfile_list) quiet maxdepth(2)
        local zipfilecount : word count `zipfile_list'
        if ( `zipfilecount' == 0 ) {
            _print_error No ZIP files to process in {input:`inputdirectory'}; cannot continue
            exit 601
        }
        _print_info Detected {input:`zipfilecount'} ZIP file`=cond(`zipfilecount'>1,"s","")' to process`=cond(${casi_popro_debug},":",".")'
        local zipcounter 0
        foreach zipfile of local zipfile_list {
            local zipfile_`++zipcounter'_filename `zipfile'
            _print_debug {result:`zipcounter'} → {input:`zipfile'}
        }

        **## extract ZIP files, check contents
        _print_info Extracting ZIP packages…
        forvalues filecounter=1/`zipcounter' {
            local unpack_dir `temp_unpack_path'/`filecounter'
            mkdir `"`unpack_dir'"'
            _extract_zip `zipfile_`filecounter'_filename' , directory(`unpack_dir')
            local maindir : dir "`unpack_dir'" dirs "*" , respectcase
            assert `: word count `maindir'' == 1
            capture : _export_name_check `maindir'
            local zipfile_`filecounter'_maindir `r(dirname)'
            if ( _rc == 0 ) {
                _print_debug export {result:`filecounter'} is from host {result:`export_host'}, performed at {result:`export_time_for_humans'}
                local zipfile_`filecounter'_hostname `export_host'
                local zipfile_`filecounter'_export_time `export_time'
                local zipfile_`filecounter'_readable_time `export_time_for_humans'
                local zipfile_`filecounter'_filename_time `export_time_for_filenames'
            }
            else {
                _print_warning Top level directory name in ZIP file {result:`filecounter'} does not seem right: {input:`zipfile_`filecounter'_maindir'}
            }
        }
    }

    **# processing data and para data
    _print_info Processing extracted files…
    forvalues dircounter=1/`zipcounter' {
        local unpack_dir `temp_unpack_path'/`dircounter'
        local dirname_date=ustrregexra(`"`zipfile_`dircounter'_readable_time'"',"[: ]","-")
        local target_dirname `zipfile_`dircounter'_filename_time'_`zipfile_`dircounter'_hostname'
        _print_info … from package {input:`dircounter'}`=cond(!missing(`"`zipfile_`dircounter'_hostname'"'),`" (from {result:`zipfile_`dircounter'_hostname'} at {result:`zipfile_`dircounter'_readable_time'})"',"")'
        **## check for TBT content
        _confirmdir `unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'
        if ( `r(directory_present)' != 0 ) {
            local zeus_logs_found 0
            _print_info … looking for ZEUS/TBT data
            local tbt_subdirectories: dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'" dirs "*" , respectcase
            local tbt_subdirectories: list tbt_subdirectories - tbt_blacklisted_dirs
            if ( !missing(`"`tbt_subdirectories'"')) {
                local ++ tbt_content_found
                local tbt_content_`tbt_content_found'_count : word count `tbt_subdirectories'
                forvalues tbt_counter=1/`tbt_content_`tbt_content_found'_count' {
                    local tbt_content_`tbt_content_found'_dir`tbt_counter': word `tbt_counter' of `tbt_subdirectories'
                    local tbt_content_`tbt_content_found'_source`tbt_counter' `unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'/`tbt_content_`tbt_content_found'_dir`tbt_counter''
                    local children_files: dir "`tbt_content_`tbt_content_found'_source`tbt_counter''" files "*", respectcase
                    if (`: word count `children_files'' != 0) {
                        foreach child_file of local children_files {
                            if ustrregexm(`"`child_file'"',"^([0-9]+)\.zip") {
                                local child_idtarget=ustrregexs(1)
                                mata : st_local("basename",pathbasename(`"`child_file'"'))
                                mata : st_local("fileext",pathsuffix(`"`child_file'"'))
                                local irregular_move_targetdir `"`tbt_content_`tbt_content_found'_source`tbt_counter''/`child_idtarget'"'
                                _print_warning … … moving irregular file {input:`child_file'} to subdirectory {result:`tbt_content_`tbt_content_found'_source`tbt_counter''/`child_idtarget'}
                                _create_maybeexistent_dir `irregular_move_targetdir'
                                local irregular_move_target `"`irregular_move_targetdir'/`basename'"'
                                local dupfile_suffix_counter 0
                                capture: confirm new file `"`irregular_move_target'"'
                                while (_rc==602) {
                                    local ++ dupfile_suffix_counter
                                    _print_warning … … … file {input:`irregular_move_target'} already exists!
                                    local irregular_move_target: subinstr local irregular_move_target `"`fileext'"' `"_`dupfile_suffix_counter'`fileext'"', all
                                    capture: confirm new file `"`irregular_move_target'"'
                                }
                                _mv `tbt_content_`tbt_content_found'_source`tbt_counter''/`child_file' `tbt_content_`tbt_content_found'_source`tbt_counter''/`child_idtarget'/
                                continue
                            }
                            _print_error … … unexpected TBT file {input:`child_file'} can not be matched to any respondent id and will remain in non-id parent directory
                        }
                    }
                    **### fill test group tracking frame
                    local tbt_id_list: dir "`tbt_content_`tbt_content_found'_source`tbt_counter''" dirs "*", respectcase
                    foreach tbt_id of local tbt_id_list {
                        confirm integer number `tbt_id'
                        frame post `frm_testgrouplist' (`""') (`tbt_id') (`"`zipfile_`dircounter'_hostname'"') (`zipfile_`dircounter'_export_time') (1) ("tbt")
                        _mv "`tbt_content_`tbt_content_found'_source`tbt_counter''/`tbt_id'" "`tbt_content_`tbt_content_found'_source`tbt_counter''/`target_dirname'/"
                    }
                }
            }
            local tbt_db_files: dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'" files "*.db" , respectcase
            local tbt_sql_files: dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'" files "*.sql" , respectcase
            local tbt_db_files : list tbt_db_files | tbt_sql_files
            if ( !missing(`"`tbt_db_files'"')) {
                if (`zeus_logs_found'==0) {
                    local ++ tbt_content_found
                    local zeus_db_packagecounter: copy local tbt_content_found
                    _create_maybeexistent_dir `unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'/ZEUS_logging
                    local tbt_counter 1
                    local tbt_content_`zeus_db_packagecounter'_count `tbt_counter'
                    local tbt_content_`zeus_db_packagecounter'_dir`tbt_counter' ZEUS_logging
                    local tbt_content_`zeus_db_packagecounter'_source`tbt_counter' `unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'/ZEUS_logging
                }
                _create_maybeexistent_dir `unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'/ZEUS_logging/`target_dirname'/
                foreach file of local tbt_db_files {
                    _mv "`unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'/`file'" "`unpack_dir'/`zipfile_`dircounter'_maindir'/`zeus_subdirectory'/ZEUS_logging/`target_dirname'/"
                }
            }
        }
        else {
            if ( missing(`"`cawimode'"') ) {
                _print_warning … no ZEUS/TBT (para) data subdirectory in ZIP file {result:`dircounter'}; this does not seem right!
            }
        }
        **## check for LimeSurvey content
        _confirmdir `unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'
        if ( `r(directory_present)' == 0 ) {
            _print_warning … no LimeSurvey (para) data subdirectory in ZIP file {result:`dircounter'}; this does not seem right!
            continue
        }
        local survey_subdirectories: dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'" dirs "`survey_subdir_stub'*" , respectcase
        foreach instrument_subdirectory of local survey_subdirectories {
            local frame_stub
            local dupvars
            local dropped_main_ids
            local dtafiles : dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`instrument_subdirectory'" files "survey_*.dta" , respectcase
            local matomofiles : dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`instrument_subdirectory'" files "matomo*.dta" , respectcase
            if (`: word count `matomofiles''>0) {
                local matomo_data_detected 1
                local dtafiles : list dtafiles | matomofiles
            }
            local exit_survey_files
            local address_survey_files
            local network_survey_files
            foreach filename of local dtafiles {
                foreach exit_survey_sid of local exit_surveys {
                    if ustrregexm(`"`macval(filename)'"',`"_`exit_survey_sid'.dta$"') {
                        local exit_survey_files : list exit_survey_files | filename
                        break
                    }
                }
                foreach address_survey_sid of local address_surveys {
                    if ustrregexm(`"`macval(filename)'"',`"_`address_survey_sid'.dta$"') {
                        local address_survey_files : list address_survey_files | filename
                        break
                    }
                }
                foreach network_survey_sid of local network_surveys {
                    if ustrregexm(`"`macval(filename)'"',`"_`network_survey_sid'.dta$"') {
                        local network_survey_files : list network_survey_files | filename
                        break
                    }
                }
            }
            local matomocount : word count `matomofiles'
            local addresssurveycount : word count `address_survey_files'
            local exitsurveycount : word count `exit_survey_files'
            local networksurveycount : word count `network_survey_files'
            local dtacount : word count `dtafiles'
            if ( `dtacount' == 0 ) {
                _print_debug No survey files in subdirectory {input:`instrument_subdirectory'}, skipped.
                continue
            }
            else if (`dtacount'-`matomocount'-`addresssurveycount'-`exitsurveycount'==0) {
                _print_debug No survey files (besides Matomo, or exit survey, or address data) in subdirectory {input:`instrument_subdirectory'}, skipped.
                continue
            }
            local dtafiles : list sort dtafiles
            local instrument_id : subinstr local instrument_subdirectory `"`survey_subdir_stub'"' "", all
            if ( missing(real(`"`instrument_id'"')) ) {
                _print_warning Could not determine instrument ID for subdirectory {input:`instrument_subdirectory'}; won't extract data from this subdirectory
                continue
            }
            _print_info … survey instrument {input:`instrument_id'}
            local frame_stub survey_instrument_`instrument_id'
            if ( ! `: list frame_stub in prepared_id_files' ) {
                local prepared_id_files : list prepared_id_files | frame_stub
                tempname frm_`frame_stub'
                frame create `frm_`frame_stub''
            }
            local first_dta_file 1
            foreach file of local dtafiles {
                local filepos : list posof "`macval(file)'" in dtafiles
                local full_target_path `unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`instrument_subdirectory'/`macval(file)'
                if ( !ustrregexm(`"`macval(file)'"',`"_([0-9]+)\.dta$"') ) {
                    _print_error Could not determine LimeSurvey SID for file {input:`full_target_path'}, file will be skipped
                    continue
                }
                local sid = ustrregexs(1)
                **### fork out exit survey dataset
                if (inlist(`sid',`: subinstr local exit_surveys " " ",", all')) {
                    if ( `filepos' == `first_dta_file') {
                        local ++ first_dta_file
                    }
                    _print_info {input}`instrument_subdirectory'/`macval(file)'{text} identified as exit survey data
                    **#### check if relevant data in the file
                    quietly : describe using `full_target_path', varlist
                    local exit_survey_varlist `r(varlist)'
                    if (`r(N)'==0) {
                        _print_warning no observations in exit survey data, will be ignored
                        continue
                    }
                    if (`: list exit_survey_required in exit_survey_varlist'!=1) {
                        _print_warning not all required variables found in exit survey data, will be ignored; missing variables are: {input:`: list exit_survey_required - exit_survey_varlist'}
                        continue
                    }
                    **#### process in extra frame
                    frame `frm_exitimportfile' {
                        use `main_idvarname' `exit_survey_required' using `"`full_target_path'"', clear
                        local ignorevarlist `main_idvarname' startdate originsid
                        unab allvars: _all
                        local contentvarlist : list allvars - ignorevarlist
                        quietly: keep if (usubstr(originsid,1,ustrlen(originsid)-3)==`"`instrument_id'"')
                        if (`c(N)')==0 {
                            _print_info no observations for survey id {input:`instrument_id'} in {input}`instrument_subdirectory'/`macval(file)'{text}, skipped
                            continue
                        }
                        quietly {
                            replace canceled="" if (canceled==`"-99 (filterbedingt fehlend)"')
                            replace timeout="" if (timeout==`"-99 (filterbedingt fehlend)"')
                            generate termination_event=.
                            label variable termination_event `"survey terminated: reason"'
                            label define termination_lbls 1 "timeout" 2 "aborted by interviewer"
                            label values termination_event termination_lbls
                            generate double termination_timestamp=clock(startdate,"YMD hms") if ( !missing(`var') )
                            format termination_timestamp %tcCCYY-NN-DD_HH:MM:SS
                            label variable termination_timestamp `"survey terminated at (timestamp)"'
                            generate termination_position=""
                            label variable termination_position `"survey terminated at (variable/item)"'
                            assert (missing(canceled)) if (!missing(timeout))
                            assert (missing(timeout)) if (!missing(canceled))
                            replace termination_event=1 if (!missing(timeout))
                            replace termination_position=cond(usubstr(timeout,1,1)=="q","item "+usubstr(timeout,2,.),timeout) if (!missing(timeout))
                            replace termination_event=2 if (!missing(canceled))
                            replace termination_position=cond(usubstr(canceled,1,1)=="q","item "+usubstr(canceled,2,.),canceled) if (!missing(canceled))
                            generate mde_id=`instrument_id', after(`main_idvarname')
                            keep `main_idvarname' mde_id termination_*
                            if ( `"`exporttimestamp'"' != `"noexporttimestamp"' ) {
                                _insert_export_variables, mode(`mode_name') host(`zipfile_`dircounter'_hostname') time(`zipfile_`dircounter'_export_time')
                            }
                            compress
                        }
                    }
                    capture: confirm frame `frm_exitdata'
                    if (_rc!=0) {
                        frame copy `frm_exitimportfile' `frm_exitdata'
                    }
                    else {
                        _vartype_compatibility , masterframe(`frm_exitdata') usingframe(`frm_exitimportfile')
                        frame `frm_exitimportfile': quietly: save `append_file', replace
                        frame `frm_exitdata' {
                            quietly : append using `append_file', nonotes
                            quietly : duplicates drop
                        }
                    }
                    continue
                }
                **### fork out matomo datasets
                if (strmatch(`"`macval(file)'"',"matomo*")) {
                    _print_info {input}`instrument_subdirectory'/`macval(file)'{text} identified as Matomo para data
                    if ( `filepos' == `first_dta_file') {
                        local ++ first_dta_file
                    }
                    _check_for_empty_dataset `full_target_path', idvar(`main_idvarname')
                    if ( `r(skip_me)' == 1) {
                        _print_info … but it does not contain any observations and will be skipped
                        continue
                    }
                    **#### process in extra frame
                    assert ustrregexm(`"`macval(file)'"',"^matomo-((events)|(visitors))")
                    local mtm_short=ustrregexs(1)
                    local frame_mtm_stub matomo_`mtm_short'_`instrument_id'
                    if ( ! `: list frame_mtm_stub in prepared_id_files' ) {
                        local prepared_id_files : list prepared_id_files | frame_mtm_stub
                        tempname frm_`frame_mtm_stub'
                        frame create `frm_`frame_mtm_stub''
                        frame `frm_`frame_mtm_stub'': quietly: append using `full_target_path', nonotes
                    }
                    else {
                        frame `frm_importfile'{
                            use `full_target_path', clear
                        }
                        _vartype_compatibility , masterframe(`frm_`frame_mtm_stub'') usingframe(`frm_importfile')
                        frame `frm_importfile'{
                            quietly : save `append_file' , replace
                        }
                        frame `frm_`frame_mtm_stub'': quietly: append using `append_file', nonotes
                    }
                    continue
                }
                **### fork out address data
                if (inlist(`sid',`: subinstr local address_surveys " " ",", all')) {
                    if ( `filepos' == `first_dta_file') {
                        local ++ first_dta_file
                    }
                    _print_info {input}`instrument_subdirectory'/`macval(file)'{text} identified as address data, and will be handled separately
                    **#### create separate unpack directories
                    local adress_unpack_dir `temp_address_unpack_path'/`dircounter'
                    _create_maybeexistent_dir `adress_unpack_dir'
                    _create_maybeexistent_dir `adress_unpack_dir'/`zipfile_`dircounter'_maindir'
                    _create_maybeexistent_dir `adress_unpack_dir'/`zipfile_`dircounter'_maindir'/`instrument_subdirectory'
                    _create_maybeexistent_dir `outputdirectory'/addressdata
                    local addressdata_fullpath `"`adress_unpack_dir'/`zipfile_`dircounter'_maindir'/`instrument_subdirectory'/`macval(file)'"'
                    **#### move SQL dump, if any, to different unpack directory
                    local db_address_dumpfile `unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/lime_address_dump.sql.gz
                    capture: confirm file `"`db_address_dumpfile'"'
                    if (_rc==0) {
                        _mv `"`db_address_dumpfile'"' `"`adress_unpack_dir'/`zipfile_`dircounter'_maindir'/"'
                    }
                    **#### move dta file to different unpack directory
                    _mv `"`full_target_path'"' `"`addressdata_fullpath'"'
                    **#### process in extra frame
                    local frame_address_stub addressdata_`instrument_id'
                    if ( ! `: list frame_address_stub in prepared_id_files' ) {
                        local prepared_id_files : list prepared_id_files | frame_address_stub
                        tempname frm_`frame_address_stub'
                        frame create `frm_`frame_address_stub''
                    }
                    frame `frm_`frame_address_stub'': quietly: append using `addressdata_fullpath', nonotes
                    continue
                }
                **### fork out network module survey data
                if (inlist(`sid',`: subinstr local network_surveys " " ",", all')) {
                    if ( `filepos' == `first_dta_file') {
                        local ++ first_dta_file
                    }
                    _print_info {input}`instrument_subdirectory'/`macval(file)'{text} identified as network module data, and will be handled explicitly
                    **#### process in extra frame
                    local frame_network_stub network_instrument_`instrument_id'
                    if ( ! `: list frame_network_stub in prepared_id_files' ) {
                        local prepared_id_files : list prepared_id_files | frame_network_stub
                        tempname frm_`frame_network_stub'
                        frame create `frm_`frame_network_stub''
                    }
                    frame `frm_network_survey_handling'{
                        use `full_target_path', clear
                        _drop_varlist_if_present `dropvars'
                        _process_network_survey, selected_value(`network_select_value') unselected_value(`network_unselected_value') varlist_clusters(`network_varlist_clusters')

                        quietly : save `append_file' , replace
                    }
                    frame `frm_`frame_network_stub'' {
                        quietly: append using `append_file', nonotes
                        if ( `id_detection_problems' == 1 ) {
                            char define _dta[id_detection_problems] 1
                        }
                    }
                    continue
                }
                _check_for_empty_dataset `full_target_path', idvar(`main_idvarname')
                if ( `r(skip_me)' == 1) {
                    if ( `filepos' == `first_dta_file') {
                        local ++ first_dta_file
                        local dupvars
                        local dropped_main_ids
                    }
                    continue
                }
                quietly : describe using `"`full_target_path'"' , varlist
                local varlist_`sid' `r(varlist)'
                local varlist_`sid' : subinstr local varlist_`sid' "`main_idvarname'" ""

                **### add new obs, if any, to idselector frame
                frame `frm_idselector' {
                    tempvar append_var
                    quietly {
                        count
                        local oldn `r(N)'
                        append using `"`full_target_path'"', nonotes keep(`main_idvarname') generate(`append_var')
                        levelsof `main_idvarname' if ( `append_var' > 0 ), local(`"new_survey_idlist"')
                        drop `append_var'
                        duplicates drop
                        local newn=`c(N)'-`oldn'
                    }
                    if (`newn'>0) {
                        _print_debug … added {result:`newn'} entries to id selector frame
                        if ( missing(`"`cawimode'"') ) {
                            **### fill test group tracking frame
                            local new_survey_idlist : list uniq new_survey_idlist
                            foreach new_survey_id of local new_survey_idlist {
                                frame post `frm_testgrouplist' (`"`new_survey_id'"') (.) (`"`zipfile_`dircounter'_hostname'"') (`zipfile_`dircounter'_export_time') (1) ("survey")
                            }
                        }
                    }
                }

                **### check if we're dealing with episodical data
                local new_episode_module 0
                _get_wide_variables `varlist_`sid'', threshold(`reshape_threshold')
                if ( `is_wide_module' == 1 ) {
                    **#### prepare episodical data
                    frame `frm_reshape_workspace' {
                        local new_episode_module 1
                        if ( `episode_module_count' > 0 ) {
                            forvalues episode_module_num = 1/`episode_module_count' {
                                local wide_stubs_union : list episode_mod_`episode_module_num'_stubs & wide_stubs
                                if ( `: word count `wide_stubs_union'' > `join_threshold' ) {
                                    local new_episode_module 0
                                    local episode_mod_`episode_module_num'_loops = max(`episode_mod_`episode_module_num'_loops', `wide_counter')
                                    local episode_mod_`episode_module_num'_stubs : list episode_mod_`episode_module_num'_stubs | wide_stubs
                                    local this_module_num : copy local episode_module_num
                                    _print_debug … matched sid {input:`sid'} to existing episodical module {result:`episode_module_num'}
                                }
                            }
                        }
                        if ( `new_episode_module' == 1 ) {
                            local ++ episode_module_count
                            local episode_mod_`episode_module_count'_stubs: copy local wide_stubs
                            local episode_mod_`episode_module_count'_loops: copy local wide_counter
                            local frame_ls_stub survey_instrument_`instrument_id'_LS`episode_module_count'
                            if ( ! `: list frame_ls_stub in prepared_id_files' ) {
                                local prepared_id_files : list prepared_id_files | frame_ls_stub
                                tempname frm_`frame_ls_stub'
                                frame create `frm_`frame_ls_stub''
                            }
                            local this_module_num : copy local episode_module_count
                            _print_debug … identified new episodical module {result:`this_module_num'} from sid {input:`sid'}
                        }
                        if ( `filepos' == `first_dta_file') {
                            local ++ first_dta_file
                        }
                        _print_debug … creating long file from sid {input:`sid'}
                        quietly : use `"`full_target_path'"' , clear
                        _drop_varlist_if_present `dropvars'
                        if ( `"`exporttimestamp'"' != `"noexporttimestamp"' ) {
                            _insert_export_variables, mode(`mode_name') host(`zipfile_`dircounter'_hostname') time(`zipfile_`dircounter'_export_time')
                        }
                        foreach reshape_varstub of local wide_stubs {
                            local varpattern : subinstr local reshape_varstub "@" "*" , all
                            unab reshape_variables : `varpattern'
                            local checked_strvars
                            local checked_numvars
                            local target_type num
                            foreach reshape_variable of local reshape_variables {
                                if ( !ustrregexm(`"`reshape_variable'"', `"w[0-9]+$"') ) continue
                                _binary_type `reshape_variable'
                                local checked_`r(binary_type)'vars : list local checked_`r(binary_type)'vars | reshape_variable
                                if (`"`target_type'"' == `"num"' & `"`r(binary_type)'"' == `"str"' ) {
                                    quietly : count if ( !missing(`reshape_variable') )
                                    if ( `r(N)' > 0 ) local target_type str
                                }
                            }
                            if ( missing(`"`checked_strvars'"') | missing(`"`checked_numvars'"') ) continue
                            if ( `"`target_type'"' == `"num"' ) {
                                foreach changevar of local checked_strvars {
                                    quietly : destring `changevar' , replace
                                }
                            }
                            else {
                                foreach changevar of local checked_numvars {
                                    quietly : tostring `changevar' , replace force
                                    quietly : replace `changevar' = "" if ( `changevar' == "." )
                                    char define `changevar'[tostring]
                                }
                            }
                        }
                        local renamesource : subinstr local wide_stubs `"@"' `""', all
                        local renametarget : subinstr local wide_stubs `"w@"' `""', all
                        local reshape_varcount : word count `renamesource'
                        forvalues varnum = 1/`reshape_varcount' {
                            local sourcevar : word `varnum' of `renamesource'
                            local reshape_`varnum'_label: variable label `sourcevar'1
                        }
                        quietly : reshape long `wide_stubs' , i(`main_idvarname') j(`spellvarname')
                        forvalues varnum = 1/`reshape_varcount' {
                            local targetvar : word `varnum' of `renamesource'
                            label variable `targetvar' `"`reshape_`varnum'_label'"'
                        }
                        if ( missing(`"`keep_empty_spells'"') ) {
                            quietly {
                                generate `row_nonmissings' = `reshape_varcount'
                                forvalues varnum = 1/`reshape_varcount' {
                                    local targetvar : word `varnum' of `renamesource'
                                    if ( usubstr(`"`: type `targetvar''"',1,3) == "str" ) {
                                        replace `row_nonmissings' = `row_nonmissings' - 1 if ( usubstr(`targetvar',1,ustrlen(`"`spell_unused_value'"')) == `"`spell_unused_value'"' )
                                    }
                                    else {
                                        replace `row_nonmissings' = `row_nonmissings' - 1 if ( `targetvar' == `spell_unused_value' )
                                    }
                                }
                                drop if ( `row_nonmissings' == 0 )
                                drop `row_nonmissings'
                            }
                        }
                        rename (`renamesource') (`renametarget')
                        _vartype_compatibility , masterframe(`frm_`frame_ls_stub'') usingframe(`frm_reshape_workspace')
                        quietly : save `append_file' , replace
                    }
                    frame `frm_`frame_ls_stub'' {
                        quietly : append using `append_file' , nonotes
                    }
                    continue
                }
                frame `frm_importfile' {
                    **## simple cross-sectional data is handled here
                    if ( `filepos' == `first_dta_file') {
                        quietly : use `"`full_target_path'"' , clear
                        _drop_varlist_if_present `dropvars'
                        if ( `"`exporttimestamp'"' != `"noexporttimestamp"' ) {
                            _insert_export_variables, mode(`mode_name') host(`zipfile_`dircounter'_hostname') time(`zipfile_`dircounter'_export_time')
                        }
                        unab dropcheck_vars : _all
                        _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                        local ignorevarlist `r(varlist)'
                        if (!missing(`"`ignorevarlist'"')) {
                            local dropcheck_vars: list dropcheck_vars - ignorevarlist
                        }
                        if ( `"`dropofemptycases'"' != `"nodropofemptycases"' ) {
                            _drop_empty_observations `dropcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing') drop_anyways(`dropped_main_ids')
                            local newly_dropped_main_ids `"`r(dropped_main_ids)'"'
                            local dropped_main_ids: list dropped_main_ids | newly_dropped_main_ids
                        }
                        continue
                    }
                    unab varlist_current : _all
                    local var_dups : list varlist_current & varlist_`sid'
                    if ( !missing(`"`var_dups'"')) {
                        local dupvars : list dupvars | var_dups
                        rename (`var_dups') (__backup_=) , r
                        local backupvars `r(newnames)'
                    }
                    **### check, and correct if possible, 1:1 relationship
                    capture : isid `main_idvarname' , missok
                    if ( _rc == 0 ) {
                        local master_mergepart 1
                    }
                    else {
                        if ( `filepos' == `first_dta_file'+1 ) {
                            _print_warning variable {input:`main_idvarname'} not uniquely identifying observations in (joint) master data; trying to de-duplicate
                            preserve
                            unab dupcheck_vars : _all
                            _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                            local ignorevarlist `r(varlist)'
                            if (!missing(`"`ignorevarlist'"')) {
                                local dupcheck_vars: list dupcheck_vars - ignorevarlist
                            }
                            _duplicates_dropper `dupcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing')
                            capture: isid `main_idvarname'
                            if ( _rc == 0 ) {
                                restore
                                local master_mergepart 1
                                quietly : _duplicates_dropper `dupcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing')
                                _print_info … successfully de-duplicated {input:`main_idvarname'}
                            }
                            else {
                                restore
                                /*
                                duplicates drop `main_idvarname', force
                                local master_mergepart 1
                                */
                                local master_mergepart m
                                _print_warning … could not de-duplicate {input:`main_idvarname'}; this will lead to non-unique data!
                            }
                        }
                        else {
                            local master_mergepart m
                        }
                    }
                    capture : isid `main_idvarname' using `"`full_target_path'"', missok
                    if ( _rc == 0 ) {
                        local using_mergepart 1
                    }
                    else {
                        _print_warning variable {input:`main_idvarname'} not uniquely identifying observations in {input:`full_target_path'}; trying to de-duplicate
                        preserve
                        quietly : use `"`full_target_path'"' , clear
                        unab dupcheck_vars : _all
                        _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                        local ignorevarlist `r(varlist)'
                        if (!missing(`"`ignorevarlist'"')) {
                            local dupcheck_vars: list dupcheck_vars - ignorevarlist
                        }
                        _duplicates_dropper `dupcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing')
                        capture: isid `main_idvarname'
                        if ( _rc == 0 ) {
                            local using_mergepart 1
                            quietly : save `"`full_target_path'"' , replace
                            _print_info … successfully de-duplicated {input:`main_idvarname'}
                        }
                        else {
                            duplicates drop `main_idvarname', force
                            local using_mergepart 1
                            quietly : save `"`full_target_path'"' , replace
                            /*
                            local using_mergepart m
                            _print_warning … could not de-duplicate {input:`main_idvarname'}; this will lead to non-unique data!
                            */
                        }
                        restore
                    }
                    local merge_mode `master_mergepart':`using_mergepart'
                    if (`"`merge_mode'"' == `"m:m"' ) {
                        _print_error The abovementioned duplicates in {input:`main_idvarname'} would lead to a {result:m:m merge} scenario; we won't do this, and cannot continue ;-(
                        exit 459
                    }
                    preserve
                    use `"`full_target_path'"' , clear
                    unab dropcheck_vars : _all
                    _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                    local ignorevarlist `r(varlist)'
                    if (!missing(`"`ignorevarlist'"')) {
                        local dropcheck_vars: list dropcheck_vars - ignorevarlist
                    }
                    if ( `"`dropofemptycases'"' != `"nodropofemptycases"' ) {
                        _drop_empty_observations `dropcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing') drop_anyways(`dropped_main_ids')
                        local newly_dropped_main_ids `"`r(dropped_main_ids)'"'
                        local dropped_main_ids: list dropped_main_ids | newly_dropped_main_ids
                        if ( `r(dropped)' > 0 ) {
                            tempfile merge_file
                            quietly : save `merge_file'
                        }
                        else {
                            local merge_file: copy local full_target_path
                        }
                    }
                    else {
                        local merge_file: copy local full_target_path
                    }
                    restore
                    /* quietly : merge `merge_mode' `main_idvarname' using `"`merge_file'"', nogenerate assert(match master) */
                    quietly : merge `merge_mode' `main_idvarname' using `"`merge_file'"', nogenerate keep(match master)
                    _drop_varlist_if_present `dropvars'
                    if ( !missing(`"`var_dups'"') ) {
                        rename (`var_dups') (=_`sid') , r
                        local ordervars `r(newnames)'
                        local aftervars `r(oldnames)'
                        rename (`backupvars') (`var_dups')
                        local backupvars
                    }
                }
            }
            if (`first_dta_file' > `: word count `dtafiles'') {
                _print_info No processable survey files in subdirectory {input:`instrument_subdirectory'}, skipped.
                continue
            }
            _vartype_compatibility , masterframe(`frm_`frame_stub'') usingframe(`frm_importfile')
            frame `frm_importfile': quietly : save `append_file' , replace
            frame `frm_`frame_stub'' {
                quietly : append using `append_file' , nonotes
                if ( !missing(`"`dupvars'"') ) {
                    foreach ordervar of local dupvars {
                        order `ordervar'_* , after(`ordervar')
                    }
                }
            }
            if ( !missing(`"`dropped_main_ids'"') ) {
                local dropped_id_count : word count `dropped_main_ids'
                local plural=cond(`dropped_id_count'>1,"s","")
                local have=cond(`dropped_id_count'>1,"have","has")
                local they=cond(`dropped_id_count'>1,"they","it")
                _print_warning {result:`dropped_id_count'} observation`plural' `have' been dropped from the data because `they' did not contain any values:
                local with_lastpage=(strmatch(`"`dropped_main_ids'"',`"*":"*"'))
                if `with_lastpage' {
                    local lastpage_name lastpage
                }
                else {
                    local lastpage_name
                }
                display as error in smcl _newline `"{result}{p2col :`main_idvarname'}`lastpage_name'{p_end}"'
                display as error in smcl `"{result}{p2line}"'
                foreach dropped_id of local dropped_main_ids {
                    if strmatch(`"`dropped_id'"',`"*":"*"') {
                        gettoken report_id rest : dropped_id, parse(":")
                        gettoken colon report_lastpage : rest, parse(":")
                        assert (`"`colon'"'==`":"')
                        local report_lastpage `report_lastpage'
                    }
                    else {
                        local report_id : copy local dropped_id
                        local report_lastpage
                    }
                    display as error in smcl `"{text}{p2col :`report_id'}`report_lastpage'{p_end}"'
                }
                display as error in smcl _newline
            }
            **## timings data
            local timingsfiles : dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`instrument_subdirectory'" files "timings_*.dta" , respectcase
            local timingsfiles : list sort timingsfiles
            local timingscount : word count `timingsfiles'
            if ( `timingscount'>0 ) {
                _print_info … exporting {result:timings} data
                local frame_stub survey_timings_`instrument_id'
                if ( ! `: list frame_stub in prepared_id_files' ) {
                    local prepared_id_files : list prepared_id_files | frame_stub
                    tempname frm_`frame_stub'
                    frame create `frm_`frame_stub''
                }
                local dupvars
                local dropped_main_ids
                local first_dta_file 1
                foreach file of local timingsfiles {
                    local filepos : list posof "`macval(file)'" in timingsfiles
                    local full_target_path `unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`instrument_subdirectory'/`macval(file)'
                    if ( !ustrregexm(`"`macval(file)'"',`"_([0-9]+)\.dta$"') ) {
                        _print_error Could not determine LimeSurvey SID for file {input:`full_target_path'}, file will be skipped
                        continue
                    }
                    local sid = ustrregexs(1)
                    **###  fork out exit survey timings dataset
                    if (inlist(`sid',`: subinstr local exit_surveys " " ",", all')) {
                        _print_info {input}`instrument_subdirectory'/`macval(file)'{text} identified as exit survey timings (those timings are irrelevant and will be skipped)
                        if ( `filepos' == `first_dta_file') {
                            local ++ first_dta_file
                        }
                        continue
                    }
                    **### fork out address data
                    if (inlist(`sid',`: subinstr local address_surveys " " ",", all')) {
                        _print_info {input}`instrument_subdirectory'/`macval(file)'{text} is address timing data and will be handled separately
                        **#### move to different unpack directory
                        local adress_unpack_dir `temp_address_unpack_path'/`dircounter'
                        _create_maybeexistent_dir `adress_unpack_dir'
                        _create_maybeexistent_dir `adress_unpack_dir'/`zipfile_`dircounter'_maindir'
                        _create_maybeexistent_dir `adress_unpack_dir'/`zipfile_`dircounter'_maindir'/`instrument_subdirectory'
                        local addressdata_fullpath `"`adress_unpack_dir'/`zipfile_`dircounter'_maindir'/`instrument_subdirectory'/`macval(file)'"'
                        if ( `filepos' == `first_dta_file') {
                            local ++ first_dta_file
                        }
                        **#### move dta file to different unpack directory
                        _mv `"`full_target_path'"' `"`addressdata_fullpath'"'
                        **#### process in extra frame
                        local frame_address_stub addresstimings_`sid'
                        if ( ! `: list frame_address_stub in prepared_id_files' ) {
                            local prepared_id_files : list prepared_id_files | frame_address_stub
                            tempname frm_`frame_address_stub'
                            frame create `frm_`frame_address_stub''
                        }
                        frame `frm_`frame_address_stub'': append using `addressdata_fullpath'
                        continue
                    }
                    _check_for_empty_dataset `full_target_path', idvar(`main_idvarname')
                    if ( `r(skip_me)' == 1) {
                        if ( `filepos' == `first_dta_file') {
                            local ++ first_dta_file
                            local dupvars
                            local dropped_main_ids
                        }
                        continue
                    }
                    quietly : describe using `"`full_target_path'"' , varlist
                    local varlist_`sid' `r(varlist)'
                    local varlist_`sid' : subinstr local varlist_`sid' "`main_idvarname'" ""
                    frame `frm_importfile' {
                        if ( `filepos' == `first_dta_file') {
                            quietly : use `"`full_target_path'"' , clear
                            _drop_varlist_if_present `dropvars'
                            if ( `"`exporttimestamp'"' != `"noexporttimestamp"' ) {
                                _insert_export_variables, mode(`mode_name') host(`zipfile_`dircounter'_hostname') time(`zipfile_`dircounter'_export_time')
                            }
                            unab dropcheck_vars : _all
                            _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                            local ignorevarlist `r(varlist)'
                            if (!missing(`"`ignorevarlist'"')) {
                                local dropcheck_vars: list dropcheck_vars - ignorevarlist
                            }
                            if ( `"`dropofemptycases'"' != `"nodropofemptycases"' ) {
                                _drop_empty_observations `dropcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing') drop_anyways(`dropped_main_ids')
                                local newly_dropped_main_ids `"`r(dropped_main_ids)'"'
                                local dropped_main_ids: list dropped_main_ids | newly_dropped_main_ids
                            }
                            continue
                        }
                        unab varlist_current : _all
                        local var_dups : list varlist_current & varlist_`sid'
                        if ( !missing(`"`var_dups'"')) {
                            local dupvars : list dupvars | var_dups
                            rename (`var_dups') (__backup_=) , r
                            local backupvars `r(newnames)'
                        }
                        **#### check, and correct if possible, 1:1 relationship
                        capture : isid `main_idvarname' , missok
                        if ( _rc == 0 ) {
                            local master_mergepart 1
                        }
                        else {
                            if ( `filepos' == `first_dta_file'+1 ) {
                                _print_warning variable {input:`main_idvarname'} not uniquely identifying observations in (joint) master data; trying to de-duplicate
                                preserve
                                unab dupcheck_vars : _all
                                _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                                local ignorevarlist `r(varlist)'
                                if (!missing(`"`ignorevarlist'"')) {
                                    local dupcheck_vars: list dupcheck_vars - ignorevarlist
                                }
                                _duplicates_dropper `dupcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing')
                                capture: isid `main_idvarname'
                                if ( _rc == 0 ) {
                                    restore
                                    local master_mergepart 1
                                    quietly : _duplicates_dropper `dupcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing')
                                    _print_info … successfully de-duplicated {input:`main_idvarname'}
                                }
                                else {
                                    restore
                                    duplicates drop `main_idvarname', force
                                    local master_mergepart 1
                                    /* local master_mergepart m
                                    _print_warning … could not de-duplicate {input:`main_idvarname'}; this will lead to non-unique data!
                                    */
                                }
                            }
                            else {
                                local master_mergepart m
                            }
                        }
                        capture : isid `main_idvarname' using `"`full_target_path'"', missok
                        if ( _rc == 0 ) {
                            local using_mergepart 1
                        }
                        else {
                            _print_warning variable {input:`main_idvarname'} not uniquely identifying observations in {input:`full_target_path'}; trying to de-duplicate
                            preserve
                            quietly : use `"`full_target_path'"' , clear
                            unab dupcheck_vars : _all
                            _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                            local ignorevarlist `r(varlist)'
                            if (!missing(`"`ignorevarlist'"')) {
                                local dupcheck_vars: list dupcheck_vars - ignorevarlist
                            }
                            _duplicates_dropper `dupcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing')
                            capture: isid `main_idvarname'
                            if ( _rc == 0 ) {
                                local using_mergepart 1
                                quietly : save `"`full_target_path'"' , replace
                                _print_info … successfully de-duplicated {input:`main_idvarname'}
                            }
                            else {
                                local using_mergepart m
                                _print_warning … could not de-duplicate {input:`main_idvarname'}; this will lead to non-unique data!
                            }
                            restore
                        }
                        local merge_mode `master_mergepart':`using_mergepart'
                        if (`"`merge_mode'"' == `"m:m"' ) {
                            _print_error The abovementioned duplicates in {input:`main_idvarname'} would lead to a {result:m:m merge} scenario; we won't do this, and cannot continue ;-(
                            exit 459
                        }
                        preserve
                        use `"`full_target_path'"' , clear
                        unab dropcheck_vars : _all
                        _get_varlist_if_present `ignore_duplicate_vars' , varlab_regex(`ignore_duplicate_varlabelregex') backup_prefix(__backup_)
                        local ignorevarlist `r(varlist)'
                        if (!missing(`"`ignorevarlist'"')) {
                            local dropcheck_vars: list dropcheck_vars - ignorevarlist
                        }
                        if ( `"`dropofemptycases'"' != `"nodropofemptycases"' ) {
                            _drop_empty_observations `dropcheck_vars' , idvar(`main_idvarname') ignorevalues(`filtered_missing') ignorestrings(`filtered_stringmissing') drop_anyways(`dropped_main_ids')
                            local newly_dropped_main_ids `"`r(dropped_main_ids)'"'
                            local dropped_main_ids: list dropped_main_ids | newly_dropped_main_ids
                            if ( `c(N)' == 0 ) {
                                _print_warning … no observations with valid information left from {input:`full_target_path'}, skipped
                                restore
                                continue
                            }
                            if ( `r(dropped)' > 0 ) {
                                tempfile merge_file
                                quietly : save `merge_file'
                            }
                            else {
                                local merge_file: copy local full_target_path
                            }
                        }
                        else {
                            local merge_file: copy local full_target_path
                        }
                        restore
                        /* quietly : merge `merge_mode' `main_idvarname' using `"`merge_file'"', nogenerate assert(match master) */
                        quietly : merge `merge_mode' `main_idvarname' using `"`merge_file'"', nogenerate keep(match master)
                        _drop_varlist_if_present `dropvars'
                        if ( !missing(`"`var_dups'"') ) {
                            rename (`var_dups') (=_`sid') , r
                            local ordervars `r(newnames)'
                            local aftervars `r(oldnames)'
                            rename (`backupvars') (`var_dups')
                            local backupvars
                        }
                    }
                }
                _vartype_compatibility , masterframe(`frm_`frame_stub'') usingframe(`frm_importfile')
                frame `frm_importfile': quietly : save `append_file' , replace
                frame `frm_`frame_stub'' {
                    quietly : append using `append_file' , nonotes
                    if ( !missing(`"`dupvars'"') ) {
                        foreach ordervar of local dupvars {
                            order `ordervar'_* , after(`ordervar')
                        }
                    }
                }
            }
        }
        **## process "classical" Matomo exports
        if (`matomo_data_detected' == 0) {
            _confirmdir `unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`matomo_subdirectory'
            if ( `r(directory_present)' == 0 ) {
                _print_warning No LimeSurvey para data in ZIP file {result:`dircounter'}; this does not seem right!
                continue
            }
            local dtafiles : dir "`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`matomo_subdirectory'" files "*.dta" , respectcase
            local dtacount : word count `dtafiles'
            if ( `dtacount' == 0 ) {
                _print_error No files in subdirectory {input:`matomo_subdirectory'}, skipped.
                continue
            }
            else if ( `dtacount' != 2 ) {
                _print_error Expected {result:2} files in subdirectory {input:`matomo_subdirectory'}, but {input:`dtacount'} present; skipped.
                continue
            }
            _print_info … survey para data
            foreach matomo_file in events visitors {
                capture : confirm file `"`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`matomo_subdirectory'/`matomo_file'.dta"'
                if ( _rc != 0 ) {
                    _print_error Expected file {result:`matomo_file'.dta} not found in subdirectory {input:`matomo_subdirectory'}, skipped.
                    continue
                }
                frame `frm_matomo_`matomo_file'' : quietly : append using `"`unpack_dir'/`zipfile_`dircounter'_maindir'/`limesurvey_subdirectory'/`matomo_subdirectory'/`matomo_file'.dta"' , nonotes
            }
        }
    }

    **# fixing non-matching tokens in Matomo data
    _print_info Fixing crumbled tokens in Matomo data…
    frame `frm_idselector' {
        quietly {
            duplicates drop
            generate lower_idvar=ustrlower(ustrtrim(`main_idvarname'))
        }
    }
    foreach export_frame of local prepared_id_files {
        if ( !missing(`"`cawimode'"') & `"`export_frame'"' == `"testgrouplist"' ) {
            continue
        }
        frame `frm_`export_frame'' : quietly : count
        if ( r(N) == 0 ) {
            continue
        }
        if (!strmatch(`"`export_frame'"',"matomo_*")) {
            continue
        }
        frame `frm_`export_frame'' {
            tempvar id_link
            capture : confirm variable `main_idvarname', exact
            if (_rc!=0) {
                local rest_idvariables: list id_variable - main_idvarname
                foreach id_candidate of local rest_idvariables {
                    capture: confirm variable `id_candidate', exact
                    if (_rc==0) {
                        rename `id_candidate' `main_idvarname'
                        continue, break
                    }
                }
            }
            confirm variable `main_idvarname', exact
            quietly : frlink m:1 `main_idvarname' , frame(`frm_idselector') generate(`id_link')
            quietly : count if missing(`id_link')
            local broken_obscount `r(N)'
            if (`broken_obscount'==0) {
                drop `id_link'
                continue
            }
            _print_debug … trying to fix {input:`main_idvarname'} for {result:`broken_obscount'} observations in {result:`export_frame'}
            quietly: levelsof `main_idvarname' if missing(`id_link'), local(broken_tokens)
            _print_debug … trying to fix `main_idvarname' for entries: {input}`broken_tokens'
            local broken_idcount: word count `broken_tokens'
            frame `frm_idselector' {
                local fixed_tokenpairs
                foreach token of local broken_tokens {
                    local searchtoken: copy local token
                    local searchtoken=ustrtrim(`"`searchtoken'"')
                    local searchtoken=ustrlower(`"`searchtoken'"')
                    local searchtoken : subinstr local searchtoken "ö" "?", all
                    quietly: count if strmatch(lower_idvar,"`searchtoken'")
                    if (`r(N)')==1 {
                        quietly: levelsof `main_idvarname' if strmatch(lower_idvar,"`searchtoken'"), local(correct_token) clean
                        _print_debug … … '`token'' `arrow' '`correct_token''
                        local tokenpair `"`"`token':`correct_token'"'"'
                        local fixed_tokenpairs: list fixed_tokenpairs | tokenpair
                    }
                    else {
                        _print_warning … … `main_idvarname' {input:`token'}: {result:`r(N)'} matches!
                    }

                }
                if (wordcount(`"`broken_tokens'"')!=wordcount(`"`fixed_tokenpairs'"')) {
                    _print_error couldn't find correct {input:`main_idvarname'} for all broken ones, please investigate!
                }
            }
            foreach tokenpair of local fixed_tokenpairs {
                local tokenpair `tokenpair'
                gettoken sourcetoken rest: tokenpair, parse(":")
                gettoken colon targettoken: rest, parse(":")
                assert (`"`colon'"'==":")
                quietly : replace token=`"`targettoken'"' if (token == `"`sourcetoken'"')
            }
            quietly : frlink rebuild `id_link'
            quietly : count if missing(`id_link')
            if (`r(N)'==0) {
                local plural=cond(`broken_idcount'>1,"s","")
                local obs_plural=cond(`broken_obscount'>1,"s","")
                _print_info … successfully repaired {result:`broken_idcount'} {input:`main_idvarname'}`plural' in a total of {result:`broken_obscount'} observation`obs_plural' in {result:`export_frame'}
            }
            else {
                _print_error There are non-matching {result:`main_idvarname'}s in {result:`export_frame'}, please investigate!
            }
            drop `id_link'
        }
    }

    **# post-processing test group tracking frame
    if ( missing(`"`cawimode'"') ) {
        _print_info Post-processing test group list…
        frame `frm_testgrouplist' {
            quietly {
                duplicates drop
                reshape wide contains_ , i(`main_idvarname' tbt_idtarget casi_export_host casi_export_time) j(survey_type "tbt" "survey") string
                replace contains_tbt=0 if ( missing(contains_tbt))
                label variable contains_tbt "observation contains tbt data"
                replace contains_survey=0 if ( missing(contains_survey))
                label variable contains_survey "observation contains survey data"
            }
        }
    }

    **# exporting concatenated data
    _print_info Exporting data…
    foreach export_frame of local prepared_id_files {
        local mde_id_from_name
        if ( !missing(`"`cawimode'"') & `"`export_frame'"' == `"testgrouplist"' ) {
            continue
        }
        frame `frm_`export_frame'' : quietly : count
        if ( r(N) == 0 ) {
            continue
        }
        if (strmatch(`"`export_frame'"',"matomo_*")) {
            local export_name : subinstr local export_frame "matomo_" "matomo-", all
            local export_subdirectory: copy local export_name
        }
        else if (strmatch(`"`export_frame'"',"survey_*") | strmatch(`"`export_frame'"',"network_*")) {
            local export_frame_stub=ustrregexra(`"`export_frame'"',"_LS[0-9]+","",1)
            local mde_id_from_name=usubstr(`"`export_frame_stub'"',ustrrpos(`"`export_frame_stub'"',"_")+1,.)
            assert (!missing(real(`"`mde_id_from_name'"')))
            local export_name: copy local export_frame
            local export_name : subinstr local export_name "survey_" "survey-", all
            local export_name : subinstr local export_name "network_" "network-", all
            if (strmatch(`"`export_frame'"',"survey_*")) {
                local export_subdirectory: copy local export_name
            }
            else {
                local export_subdirectory: subinstr local export_name "network-" "survey-"
            }
        }
        else {
            local export_name: copy local export_frame
            local export_subdirectory: copy local export_name
        }
        if ustrregexm(`"`export_subdirectory'"',`"(.*)_[0-9]+(_LS[0-9+])?$"') {
            local export_subdirectory=ustrregexs(1)
        }
        local export_subdirectory `studynumber'_`export_subdirectory'
        _print_info … {result:`export_subdirectory' :: `export_name'}
        frame `frm_`export_frame'' {
            foreach other_idvarname of local other_idvarnames {
                capture : confirm variable `other_idvarname' , exact
                if ( _rc == 0 ) {
                    capture : confirm variable `main_idvarname' , exact
                    if ( _rc != 0 ) {
                        rename `other_idvarname' `main_idvarname'
                    }
                }
            }
            capture : confirm variable `main_idvarname' , exact
            if ( _rc == 0 ) order `main_idvarname' , first

            **## add exit survey content, if applicable (only for LimeSurvey surveydata exports)
            capture: confirm frame `frm_exitdata'
            if (_rc==0 & !missing(`"`mde_id_from_name'"')) {
                _print_info … … adding exit survey information
                tempvar id_link mde_id
                quietly {
                    generate `mde_id'=`mde_id_from_name'
                    capture: frlink m:1 `main_idvarname' `mde_id' casi_export_host casi_export_time, frame(`frm_exitdata' `main_idvarname' mde_id casi_export_host casi_export_time) generate(`id_link')
                    if (_rc!=0) {
                        clonevar mde_id=`mde_id'
                        frame copy `c(frame)' master_frame
                        frame copy `frm_exitdata' using_frame
                        di as err `"failed: frlink m:1 `main_idvarname' mde_id, frame(using_frame `main_idvarname' mde_id) generate(`id_link')"'
                        exit 459
                    }
                    _get_order_opt
                    frget * , from(`id_link') exclude(`main_idvarname')
                    order `r(newlist)', `order_opt'
                }
                drop `id_link' `mde_id'
            }

            **## remove test tokens, if any
            if ( !missing(`"`test_observation_regex'"') ) {
                tempvar test_observation
                if ( usubstr(`"`: type `main_idvarname''"',1,3)=="str") {
                    local test_string `main_idvarname'
                }
                else {
                    local test_string strofreal(`main_idvarname',"%30.0g")
                }
                quietly : generate `test_observation'=( ustrregexm(`test_string',`"`macval(test_observation_regex)'"') )
                quietly : count if (`test_observation')
                if (`r(N)'>0) {
                    local erase_obscount `r(N)'
                    quietly: distinct `main_idvarname' if (`test_observation')
                    local erase_idcount `r(ndistinct)'
                    quietly: levelsof `main_idvarname' if (`test_observation'), local(test_ids) clean
                    _print_info erasing {result:`erase_obscount'} observation`=cond(`erase_obscount'>1,"s","")' for test id`=cond(`erase_idcount'>1,"s","")': {result:`test_ids'}
                }
                quietly : drop if (`test_observation')
                drop `test_observation'
            }

            **## insert target id variable, if we have one
            if ( !missing(`"`id_list'"')) {
                capture : confirm new variable `target_idvarname' , exact
                if ( _rc != 0 ) {
                    if ( !missing(`"`target_idvarname_replace'"')) {
                        _print_info … variable {input:`target_idvarname'} will be replaced.
                        drop `target_idvarname'
                    }
                    else {
                        _print_error Variable {input:`target_idvarname'} already exists; use option {input}{opt idlist(… , replace)}{error} to overwrite.
                        exit 459
                    }
                }
                tempvar id_link
                quietly : frlink m:1 `main_idvarname' , frame(`frm_id_list') generate(`id_link')
                local unmatched = cond(missing(`"`r(unmatched)'"'), 0, `r(unmatched)')
                quietly : frget `target_idvarname' `kill_switch_var', from(`id_link')
                quietly: count if (`kill_switch_var'==1)
                if (`r(N)'>0) {
                    local erase_obscount `r(N)'
                    quietly: distinct `main_idvarname' if (`kill_switch_var'==1)
                    local erase_idcount `r(ndistinct)'
                    _print_info erasing {result:`erase_obscount'} observation`=cond(`erase_obscount'>1,"s","")' for {result:`erase_idcount'} black listed id`=cond(`erase_idcount'>1,"s","")'
                }
                drop `kill_switch_var'
                order `target_idvarname' , after(`main_idvarname')
                local plural = cond(`unmatched'>1,"s","")
                if ( `"`export_frame'"' == `"testgrouplist"' ) {
                    quietly {
                        replace `target_idvarname' = tbt_idtarget if ( missing(`target_idvarname') & !missing(tbt_idtarget) )
                        foreach element in tbt survey {
                            tempvar contains_`element'_max
                            bysort `target_idvarname' casi_export_time casi_export_host : egen `contains_`element'_max'=max(contains_`element')
                            replace contains_`element'=`contains_`element'_max'
                            drop `contains_`element'_max'

                        }
                        duplicates drop `target_idvarname' casi_export_time casi_export_host, force
                        count if ( missing(`target_idvarname') )
                        local unmatched `r(N)'
                        if ( `unmatched' > 0 ) {
                            replace `main_idvarname' = strofreal(tbt_idtarget) if ( missing(`main_idvarname') & !missing(tbt_idtarget) )
                        }
                        drop tbt_idtarget
                    }
                }
                local unmatched_warning_addition
                if ( `unmatched' == 0 ) {
                    if ( missing(`"`keepsource'"') ) {
                        drop `main_idvarname'
                    }
                }
                if ( `unmatched' > 0 ) {
                    local what_happened_to_missings "missing value`plural'"
                    local unmatched_warning_addition `" and {input:`main_idvarname'} will {bf:not} be removed"'
                    if ( `"`fakeids'"' != `"nofakeids"') {
                            preserve
                            quietly : keep if ( missing(`id_link') )
                            keep `main_idvarname'
                            quietly : duplicates drop
                            quietly : count
                            local unique_fake_ids `r(N)'
                            _print_warning Creating {result:`unique_fake_ids'} fake id entr`=cond(`unique_fake_ids'>1,"ies","y")' for {input:`target_idvarname'} in id list due to unmatched observation`plural':
                            display as error in smcl _newline `"{result}{p2col :`main_idvarname'}generated {input:`target_idvarname'}{p_end}"'
                            display as error in smcl `"{result}{p2line}"'
                            forvalues obscounter = 1/`c(N)' {
                                local add_id = `main_idvarname'[`obscounter']
                                frame `frm_id_list': _insert_new_idpair `"`add_id'"' , primaryname(`main_idvarname') secondaryname(`target_idvarname')
                            }
                            display as error in smcl _newline
                            restore
                            local what_happened_to_missings `"{result:`unique_fake_ids'} fake id`=cond(`unique_fake_ids'>1,"s","")'"'
                            quietly: frlink rebuild `id_link'
                            local unmatched_after_rebuild = cond(missing(`"`r(unmatched)'"'), 0, `r(unmatched)')
                            assert ( `unmatched_after_rebuild' == 0 )
                            drop `target_idvarname'
                            quietly : frget `target_idvarname' , from(`id_link')
                            order `target_idvarname' , after(`main_idvarname')
                            if ( `"`export_frame'"' == `"testgrouplist"' ) {
                                quietly {
                                    foreach element in tbt survey {
                                        tempvar contains_`element'_max
                                        bysort `target_idvarname' casi_export_time casi_export_host : egen `contains_`element'_max'=max(contains_`element')
                                        replace contains_`element'=`contains_`element'_max'
                                        drop `contains_`element'_max'
                                    }
                                    duplicates drop `target_idvarname' casi_export_time casi_export_host, force
                                }
                            }
                            if ( missing(`"`keepsource'"') ) {
                                drop `main_idvarname'
                            }
                    }
                    else {
                        _print_warning {result:`unmatched'} observation`plural' without matching ID`plural' in id list; {input:`target_idvarname'} will contain `what_happened_to_missings'`unmatched_warning_addition'!
                    }
                }
                drop `id_link'
                if strmatch(`"`export_frame'"',"network_*") {
                    local id_detection_problems: char _dta[id_detection_problems]
                    if `id_detection_problems' {
                        tempvar institution_id
                        generate `institution_id'=real(usubstr(strofreal(`target_idvarname',"%30.0g"),1,7))
                        local clusters `network_varlist_clusters'
                        foreach cluster_varlist of local clusters {
                            foreach namevar in `cluster_varlist' {
                                capture: confirm variable `namevar', exact
                                if (_rc==0) {
                                    quietly: replace `namevar' = (`institution_id'*10000)+`namevar' if (`institution_id'>0 & `namevar'!=`network_unselected_value' & `namevar'>0 & !missing(`namevar'))
                                }
                            }
                        }
                        drop `institution_id'
                        _print_warning re-constructed target ids from network survey module; please double check result!
                    }
                }

            }
            **## prettify LimeSurvey technical variables
            _prettify_limesurvey_techvars
            quietly : compress
            if (strmatch(`"`export_frame'"',"*address*")) {
                _create_maybeexistent_dir `outputdirectory'/addressdata/`export_subdirectory'
                local targetname `"`outputdirectory'/addressdata/`export_subdirectory'/`studynumber'_`focusperson'_`export_name'"'
            }
            else {
                _create_maybeexistent_dir `outputdirectory'/`export_subdirectory'
                local targetname `"`outputdirectory'/`export_subdirectory'/`studynumber'_`focusperson'_`export_name'"'
            }
            if ( !missing(`"`version'"') ) {
                local targetname `targetname'_`version'
            }
            quietly : save `"`targetname'.dta"' , `replace'
        }
    }

    **# copying TBT data
    if ( `tbt_content_found' > 0 ) {
        _print_info Copying TBT files…
        _create_maybeexistent_dir `outputdirectory'/`studynumber'_TBT
        forvalues pkgcounter = 1/`tbt_content_found' {
            forvalues tbtcounter = 1/`tbt_content_`pkgcounter'_count' {
                _print_debug … {result:`tbt_content_`pkgcounter'_dir`tbtcounter''}
                _recurse_and_copy_dir , from(`tbt_content_`pkgcounter'_source`tbtcounter'') to(`outputdirectory'/`studynumber'_TBT/`tbt_content_`pkgcounter'_dir`tbtcounter'')
            }
        }
    }

    **# packing and copying raw data archive
    local oldpwd `c(pwd)'
    local addressdata_subdirs: dir "`temp_address_unpack_path'" dirs "*", respectcase
    local addressdata_files: dir "`temp_address_unpack_path'" files "*", respectcase
    if (!missing(`"`addressdata_subdirs'"') | !missing(`"`addressdata_files'"')) {
        _print_info Creating ZIP raw archive with {result:address data} for archival purposes…
        noisily display as error in smcl _newline
		noisily display as error in smcl `"{tab}{bf}{dup 60:=}"'
		noisily display as error in smcl `"{tab}"' _skip(1) `"__"' _skip(10) `"__"' _skip(14) `"_"'
		noisily display as error in smcl `"{tab}"' _skip(1) `"\ \"' _skip(8) `"/ /"' _skip(13) `"(_)"'
		noisily display as error in smcl `"{tab}"' _skip(2) `"\ \"' _skip(2) `"/\"' _skip(2) `"/ /_ _ _ __ _ __  _ _ __   __ _"'
		noisily display as error in smcl `"{tab}"' _skip(3) `"\ \/  \/ / _\` | '__| '_ \| | '_ \ / _\` |"'
		noisily display as error in smcl `"{tab}"' _skip(4) `"\"' _skip(2) `"/\"' _skip(2) `"/ (_| | |"' _skip(2) `"| | | | | | | | (_| |"'
		noisily display as error in smcl `"{tab}"' _skip(5) `"\/"' _skip(2) `"\/ \__,_|_|"' _skip(2) `"|_| |_|_|_| |_|\__, |"'
		noisily display as error in smcl `"{tab}"' _skip(38) `"__/ |"'
		noisily display as error in smcl `"{tab}"' _skip(37) `"|___/"'
		noisily display as error in smcl `"{tab}{bf}{dup 60:=}"'
		noisily display as error in smcl _newline
		noisily display as error in smcl `"{tab}DO NOT DELIVER THE CONTENT OF {input:`outputdirectory'/addressdata/}"'
		noisily display as error in smcl `"{tab}TO LIFBI OR NEPS NETWORK PARTNERS, IT LIKELY CONTAINS PERSONAL DATA OF RESPONDENTS."'
		noisily display as error in smcl `"{tab}Please treat it as confidential as appropriate."'
        noisily display as error in smcl _newline
        _create_maybeexistent_dir `outputdirectory'/addressdata/archive , replace
        nobreak {
            quietly {
                local zipname `studynumber'_`focusperson'_addressdata_`mode_name'.zip
                cd `"`temp_address_unpack_path'"'
                mata: st_local("resolved_path", pathresolve("`oldpwd'", "`outputdirectory'/addressdata/archive/`zipname'"))
                zipfile `addressdata_files' `addressdata_subdirs' , saving(`"`resolved_path'"')
                cd `"`oldpwd'"'
            }
        }
    }
    _print_info Creating ZIP raw archive for archival purposes…
    _create_maybeexistent_dir `outputdirectory'/archive , replace
    nobreak {
        quietly {
            local zipname `studynumber'_`focusperson'_rawdata_`mode_name'.zip
            cd `"`temp_unpack_path'"'
            local parent_dirs :  dir "." dirs "*", respectcase
            local files
            local dirs
            foreach parent_dir of local parent_dirs {
                local add_files : dir "./`parent_dir'" files "*", respectcase
                if ( !missing(`"`add_files'"')) {
                    foreach file of local add_files {
                        local add_file "./`parent_dir'/`file'"
                        local files : list files | add_file
                    }
                }
                local add_dirs  : dir "./`parent_dir'" dirs "*", respectcase
                if ( !missing(`"`add_dirs'"')) {
                    foreach dir of local add_dirs {
                        local add_dir "./`parent_dir'/`dir'"
                        local dirs : list dirs | add_dir
                    }
                }
            }
            mata: st_local("resolved_path", pathresolve("`oldpwd'", "`outputdirectory'/archive/`zipname'"))
            zipfile `files' `dirs' , saving(`"`resolved_path'"')
            cd `"`oldpwd'"'
        }
    }

    **# done
    if ( `"`cleanup'"' != `"nocleanup"') {
        _print_info Cleaning up temporary directories.
        **## give the file system some time to rest after the hard job
        sleep 2000
        _recurse_and_erase_dir `temp_unpack_path'
        _recurse_and_erase_dir `temp_address_unpack_path'
    }
    macro drop casi_popro*
    frames reset
    clear
    _print_info 🎉 We're done processing `upper_mode_name' files! 🎉
    exit 0
end

**# sub-programs
program define _print_info, nclass
    noisily: display as result in smcl `"{p 2 8 2 0}{txt}{bf:INFO: }`0'{p_end}"'
    exit 0
end

program define _print_debug, nclass
    if ( ${casi_popro_debug} ) {
        noisily: display as error in smcl `"{p 2 9 2 0}{txt}{bf:DEBUG: }`0'{p_end}"'
    }
    exit 0
end

program define _print_warning, nclass
    noisily: display as error in smcl `"{p 2 11 2 0}{err}{bf:WARNING: }{txt}`0'{p_end}"'
    exit 0
end

program define _print_error, nclass
    noisily: display as error in smcl `"{p 2 9 2 0}{err}{bf:ERROR: }`0'{p_end}"'
    exit 0
end

program define _check_ado_presence, nclass
    foreach progname in `0' {
        capture : which `progname'
        if (_rc != 0) {
            _print_error Command {cmd:`progname'} not found, but it is required by {cmd:casi_popro}. Please {net search `progname':install first}, and try again.
            exit 601
        }
    }
    exit 0
end

program define _create_maybeexistent_dir, nclass
    syntax anything(everything id="directory name" name=dirname) [, replace]
    local replace = !missing(`"`replace'"')
    nobreak {
        _confirmdir `macval(dirname)'
        if ( `r(directory_present)' & `replace' != 1 ) {
            _print_debug Directory {input:`macval(dirname)'} present, nothing to do.
            exit 0
        }
        if ( `r(directory_present)' == 1 & `replace' == 1 ) {
            _print_debug Directory {input:`macval(dirname)'} present, re-creating it.
            _recurse_and_erase_dir `macval(dirname)'
        }
        else {
            _print_debug Directory {input:`macval(dirname)'} not present, creating it.
        }
        mkdir `macval(dirname)'
    }
end

program define _recurse_and_copy_dir, nclass
    syntax , from(string) to(string)
    _create_maybeexistent_dir `to'
    local subdir_list : dir "`macval(from)'" dirs "*" , respectcase
    foreach subdir of local subdir_list {
        _recurse_and_copy_dir , from(`macval(from)'/`macval(subdir)') to(`macval(to)'/`macval(subdir)')
    }
    local file_list : dir "`macval(from)'" files "*" , respectcase
    foreach file of local file_list {
        quietly : copy `"`macval(from)'/`macval(file)'"' `"`macval(to)'/"', replace
    }
end

program define _mv, nclass
    syntax anything(name=filespec id="files specifications") [, replace]
    gettoken sourcepath rest : filespec
    gettoken targetpath empty : rest
    if (!missing(`"`empty'"') | inlist("",`"`0'"',`"`sourcepath'"',`"`targetpath'"')) {
        local result=cond(missing(`"`0'"')," None", `": {input}`0'"')
        _print_error two arguments {input:sourcepath} and {input:targetpath} allowed for {cmd:_mv}; you provided`result'
        exit 198
    }
    _confirmdir `macval(sourcepath)'
    local source_is_directory `r(directory_present)'
    if ( `source_is_directory' ) {
        local mata_sourcepath `sourcepath'
        local targetpath `targetpath'
        local pathseparator=cond(inlist(usubstr(`"`targetpath'"',-1,.),"/","\"),"","/")
        mata : st_local("basename",pathbasename(`"`mata_sourcepath'"'))
        _create_maybeexistent_dir `targetpath'
        _create_maybeexistent_dir `targetpath'`pathseparator'`basename'
        nobreak {
            capture : _recurse_and_copy_dir, from(`macval(sourcepath)') to(`targetpath'`pathseparator'`basename'/)
        }
    }
    else {
        confirm file `macval(sourcepath)'
        nobreak {
            capture: copy `macval(sourcepath)' `macval(targetpath)'
        }
    }
    if _rc == 0  {
        if ( `source_is_directory' ) {
            _recurse_and_erase_dir `macval(sourcepath)'
        }
        else {
            rm `macval(sourcepath)'
        }
    }
    else {
        _print_error could not copy {input}`sourcepath' to {input}`targetpath'
        error `c(rc)'
    }
    exit 0
end

program define _recurse_and_erase_dir, nclass
    syntax anything(everything id="directory name" name=dirname)
    local subdir_list : dir "`macval(dirname)'" dirs "*" , respectcase
    foreach subdir of local subdir_list {
        _recurse_and_erase_dir `macval(dirname)'/`macval(subdir)'
    }
    local file_list : dir "`macval(dirname)'" files "*" , respectcase
    foreach file of local file_list {
        capture : rm `"`macval(dirname)'/`macval(file)'"'
        if ( _rc != 0) {
            _print_debug … could not erase {input:`macval(dirname)'/`macval(file)'}, retrying after 1s wait
            sleep 1000
            rm `"`macval(dirname)'/`macval(file)'"'
        }
    }
    capture : rmdir `"`macval(dirname)'"'
    if ( _rc != 0) {
        _print_debug … could not erase {input:`macval(dirname)'}, retrying after 1s wait
        sleep 1000
        rmdir `"`macval(dirname)'"'
    }
end

program define _set_default_option, nclass
    syntax name(name=name id="option name") , default(string asis)
    _print_debug Option {opt `name'} not specified, using default {input:`default'}
    c_local `name' `"`default'"'
    exit
end

program define _extract_zip , nclass
    syntax anything(everything id="ZIP file name" name=zipfile) , directory(string)
    _print_debug extracting {input:`zipfile'} to directory {input:`directory'}.
    local workdir `c(pwd)'
    mata : st_local("fullzippath",pathresolve(st_local("workdir"),st_local("zipfile")))
    nobreak {
        quietly : cd `"`directory'"'
        quietly : unzipfile `fullzippath'
        quietly : cd `"`workdir'"'
    }
    exit 0
end

program define _confirmdir , rclass
    syntax anything(everything id="directory name" name=dirname)
    nobreak {
        local workdir `"`c(pwd)'"'
        capture : cd `"`macval(dirname)'"'
        if _rc == 0 {
            quietly : cd `"`workdir'"'
            return scalar directory_present = 1
        }
        else {
            return scalar directory_present = 0
        }
    }
    exit 0
end

program define _export_name_check , rclass
    syntax anything(everything id="directory name" name=dirname)
    local dirname `macval(dirname)'
    return local dirname `macval(dirname)'
    if ( ustrregexm(`"`macval(dirname)'"',`"^ca[ws]i_export_([\-a-zA-Z0-9]+)_([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}(-[0-9]{2})?)$"') ){
        c_local export_host = ustrregexs(1)
        if !missing(ustrregexs(3)) local pattern YMD hms
        else local pattern YMD hm
        c_local export_time = clock(ustrregexs(2),`"`pattern'"')
        c_local export_time_for_humans : display %tc clock(ustrregexs(2),`"`pattern'"')
        c_local export_time_for_filenames : display %tcCCYY-NN-DD-HH-MM-SS clock(ustrregexs(2),`"`pattern'"')
        exit 0
    }
    exit 459
end

program define _get_varlist_if_present, rclass
    syntax anything, [varlab_regex(string)  backup_prefix(string)]
    local returnlist
    unab all_vars: _all
    foreach dropvar_spec of local anything {
        capture : unab dropvarlist : `dropvar_spec'
        if ( _rc == 0 ) {
            local returnlist : list returnlist | dropvarlist
        }
        if ( !missing(`"`backup_prefix'"') ) {
            capture : unab dropvarlist : `backup_prefix'`dropvar_spec'
            if ( _rc == 0 ) {
                local returnlist : list returnlist | dropvarlist
            }
        }
    }
    if ( !missing(`"`varlab_regex'"') ) {
        foreach checkvar of local all_vars {
            if ( !ustrregexm(`"`: variable label `checkvar''"', `"`macval(varlab_regex)'"') ) {
                continue
            }
            local returnlist : list returnlist | checkvar
        }
    }
    /* this is a more or less dirty workaround for derived auxiliary variables in LimeSurvey */
    local aux_addons
    foreach varname of local returnlist {
        local aux_namecandidate=usubstr(`"`varname'"',1,ustrlen(`"`varname'"')-1)+"len"
        capture : confirm variable `aux_namecandidate', exact
        if ( _rc==0 ) {
            local aux_addons : list aux_addons | aux_namecandidate
        }
    }
    local returnlist : list returnlist | aux_addons
    return local varlist `"`returnlist'"'
    exit 0
end

program define _drop_varlist_if_present, nclass
    syntax anything
    _get_varlist_if_present `anything'
    if ( !missing(`"`r(varlist)'"') ) {
        drop `r(varlist)'
    }
    exit 0
end

program define _binary_type , rclass
    syntax varname
    local vartype : type `varlist'
    if ( usubstr(`"`vartype'"',1,3) == "str" ) {
        local vartype str
    }
    else {
        local vartype num
    }
    return local binary_type `"`vartype'"'
    exit 0
end

program define _parse_id_list_opt, nclass
    syntax anything(everything id="file path" name=id_list_path) [, keepsource replace nofakeids]
    c_local id_list_path `id_list_path'
    c_local keepsource `keepsource'
    c_local fakeids `fakeids'
    c_local target_idvarname_replace `replace'
    exit 0
end

program define _get_wide_variables , nclass
    syntax namelist(id="wide test variables" name=allvars), threshold(integer)
    local wide_stubs
    local no_reshape_vars
    local wide_counter 0
    local regex_pattern `"^(.*)w([0-9]+)$"'
    foreach testvar of local allvars {
        local is_wide 0
        local widestub
        local wide_varcount 0
        if ustrregexm(`"`testvar'"', `"`macval(regex_pattern)'"') {
            local wide_stem=ustrregexs(1)
            local wide_stub=`"`wide_stem'w@"'
            local wide_varpattern=`"`wide_stem'w*"'
            foreach sibling_var of local allvars {
                if (ustrregexm(`"`sibling_var'"',`"^`wide_stem'w[0-9]+$"')) {
                local ++ wide_varcount
            }
	    }
            if ( `wide_varcount' > `threshold' ) {
                local is_wide 1
            }
        }
        if ( `is_wide' == 1 ) {
            local wide_stubs : list wide_stubs | wide_stub
            local wide_counter = max(`wide_counter',`wide_varcount')
        }
        else {
            local no_reshape_vars : list no_reshape_vars | testvar
        }
    }
    c_local wide_counter `wide_counter'
    c_local wide_stubs `wide_stubs'
    c_local no_reshape_vars `no_reshape_vars'
    c_local is_wide_module=(`wide_counter'>`threshold')
end

program define _insert_new_idpair, nclass
    syntax anything(id="primary id value" name=id), primaryname(varname) secondaryname(varname)
    local id `id'
    display as error in smcl `"{text}{p2col :`id'}"' _continue
    local new_id_dup = 1
    while ( `new_id_dup' == 1 ) {
        local created_id=runiformint(-999999999,-900000000)
        quietly : count if `secondaryname' == `created_id'
        local new_id_dup = `r(N)'
    }
    nobreak {
        quietly {
            set obs `=`c(N)'+1'
            if ( usubstr(`"`: type `primaryname''"',1,3) == "str" ) {
                replace `primaryname'=`"`macval(id)'"' in l
            }
            else {
                replace `primaryname'=`id' in l
            }
            replace `secondaryname'=`created_id' in l
        }
        display as error in smcl `"{text}`created_id'{p_end}"'
    }
    exit 0
end

program define _vartype_compatibility, nclass
    syntax , masterframe(name) usingframe(name)
    frame `usingframe' {
        quietly : count
        if ( `r(N)' == 0 ) {
            _print_error … no observations in using dataset, cannot append data; this seems wrong!
            exit 459
        }
        unab using_vars : _all
        local using_types
        foreach using_var of local using_vars {
            _binary_type `using_var'
            local next_type `r(binary_type)'
            local using_types `using_types' `next_type'
        }
    }
    frame `masterframe' : quietly : count
    if ( `r(N)' > 0) {
        frame `masterframe' {
            unab master_vars : _all
            local master_types
            foreach master_var of local master_vars {
                _binary_type `master_var'
                local next_type `r(binary_type)'
                local master_types `master_types' `next_type'
            }
        }
        foreach master_var of local master_vars {
            if ( ! `: list master_var in using_vars') {
                continue
            }
            local master_pos : list posof "`master_var'" in master_vars
            local master_type : word `master_pos' of `master_types'
            local using_pos : list posof "`master_var'" in using_vars
            local using_type : word `using_pos' of `using_types'
            if ( `"`master_type'"' == `"`using_type'"') {
                continue
            }
            frame `masterframe' {
                quietly : count if !missing(`master_var')
                local master_nonmissing `r(N)'
            }
            frame `usingframe' {
                quietly : count if !missing(`master_var')
                local using_nonmissing `r(N)'
            }
            if ( `master_nonmissing' == 0 | `using_nonmissing' == 0 ) {
                if ( `master_nonmissing' == 0 & `using_nonmissing' == 0 ) {
                    if ( `"`master_type'"' == "str" ) {
                        frame `masterframe': quietly : destring `master_var' , replace
                    }
                    else {
                        frame `usingframe': quietly : destring `master_var' , replace
                    }
                }
                else if ( `master_nonmissing' == 0 & `"`using_type'"' == "str" ) {
                    frame `masterframe' {
                        quietly : tostring `master_var' , replace force
                        quietly : replace `master_var' = "" if ( `master_var' == "." )
                        char define `master_var'[tostring]
                    }
                }
                else if ( `master_nonmissing' == 0 & `"`using_type'"' == "num" ) {
                    frame `masterframe': quietly : destring `master_var' , replace
                }
                else if ( `using_nonmissing' == 0 & `"`master_type'"' == "str" ) {
                    frame `usingframe' {
                        quietly : tostring `master_var' , replace force
                        quietly : replace `master_var' = "" if ( `master_var' == "." )
                        char define `master_var'[tostring]
                    }
                }
                else if ( `using_nonmissing' == 0 & `"`master_type'"' =="num" ) {
                    frame `usingframe': quietly : destring `master_var' , replace
                }
            }
            else {
                _print_warning Variable {input:`master_var'} has conflicting variable types between data exports; will be force-converted to string!
                if ( `"`master_type'"' == "num" ) {
                    frame `masterframe' {
                        quietly : tostring `master_var' , replace force
                        quietly : replace `master_var' = "" if ( `master_var' == "." )
                        char define `master_var'[tostring]
                    }
                }
                else {
                    frame `usingframe' {
                        quietly : tostring `master_var' , replace force
                        quietly : replace `master_var' = "" if ( `master_var' == "." )
                        char define `master_var'[tostring]
                    }
                }
            }
        }
    }
    exit 0
end

program define _insert_export_variables, nclass
    syntax , mode(name) [host(string) time(string)] [nogenerate]
    if ( `"`generate'"' != `"nogenerate"') {
        foreach option in host time {
            capture: confirm existence ``option''
            if ( _rc!=0 ) {
                _print_error option {input:`option'} required unless option {input:`nogenerate'} is specified
                exit 198
            }
        }
        _get_order_opt
    }
    else {
        foreach option in host time {
            capture: confirm existence ``option''
            if ( _rc==0 ) {
                _print_debug option {input:`option'} ignored, because option {input:`nogenerate'} is specified
                exit 198
            }
        }
    }
    local upper_mode_name=ustrupper(`"`mode'"')
    quietly {
        if ( `"`generate'"' != `"nogenerate"') {
            generate `mode'_export_host = `"`host'"', `order_opt'
        }
        label variable `mode'_export_host `"NEPS `upper_mode_name' export host name"'
        if ( `"`generate'"' != `"nogenerate"') {
            generate double `mode'_export_time = `time', after(`mode'_export_host)
        }
        format `mode'_export_time %tc
        label variable `mode'_export_time `"NEPS `upper_mode_name' export time stamp"'
    }
    exit 0
end

program define _get_order_opt, nclass
    capture: confirm variable datestamp, exact
    if (_rc==0) {
        local order_opt after(datestamp)
    }
    else {
        capture: confirm variable duration_interview
        if (_rc==0) {
            local order_opt before(duration_interview)
        }
        else {
            confirm variable token
            local order_opt after(token)
        }
    }
    c_local order_opt `order_opt'
    exit 0
end

program define _duplicates_dropper, nclass sortpreserve
    syntax varlist , idvar(varname) ignorevalues(numlist integer missingokay) ignorestrings(string asis)
    tempvar misscount dupmarker dupcounter dropmarker obs_dropcount obs_n
    **## determine variable list to check
    local checkvars : list varlist - idvar
    local checkvarcount : word count `checkvars'
    _print_info duplicate check for {input:`checkvarcount'} variables
    _print_debug {input:`checkvars'}
    local ignorevalues_comma : subinstr local ignorevalues " " "," , all
    **## use duplicates drop as a first start
    local oldn `c(N)'
    quietly : duplicates drop
    local newn `c(N)'
    local dropped=`oldn'-`newn'
    local plural=cond(`dropped'>1,"s","")
    if ( `dropped' > 0 ) {
        _print_info dropped {result:`dropped'} redundant observation`plural'
    }
    **## check if this was enough
    capture : isid `idvar'
    if ( _rc==0 ) {
        exit 0
    }
    **## remove completely missing observations
    quietly {
        duplicates tag `idvar' , generate(`dupcounter')
        generate `dupmarker' = ( `dupcounter' > 0 )
        generate `misscount' = 0 if ( `dupmarker' == 1 )
        foreach checkvar of local checkvars {
            if ( usubstr(`"`: type `checkvar''"',1,3) == "str" ) {
                quietly: replace `misscount'=`misscount'+1 if ( `dupmarker' == 1 & inlist(`checkvar',`macval(ignorestrings)') )
            }
            else {
                quietly: replace `misscount'=`misscount'+1 if ( `dupmarker' == 1 & inlist(`checkvar',`macval(ignorevalues_comma)') )
            }
        }
        generate `dropmarker' = ( `dupmarker'==1 & `misscount' == `checkvarcount' )
        **### avoid dropping all observations for an id that contains completely missing values in all rows
        sort `idvar'
        by `idvar' : egen `obs_dropcount' = total(`dropmarker')
        by `idvar' : generate `obs_n' = _N
        by `idvar' : replace `dropmarker' = 0 if (_n == 1 & `obs_dropcount' == `obs_n')
        count if ( `dropmarker' == 1 )
        local dropped `r(N)'
        if (`dropped' > 0) {
            drop if ( `dropmarker' == 1 )
        }
    }
    local plural=cond(`dropped'>1,"s","")
    if ( `dropped' > 0 ) {
        _print_info dropped {result:`dropped'} "duplicate" observation`plural' without valid values
    }
    **## check if this was enough
    capture : isid `idvar'
    if ( _rc==0 ) {
        exit 0
    }
    **## complex pairwise check for redundancy of two observations
    ** TODO, AS SOON AS WE NEED IT **
    exit 0
end

program define _drop_empty_observations, rclass
    syntax varlist, idvar(varname) ignorevalues(numlist integer missingokay) ignorestrings(string asis) [drop_anyways(string asis)]
    tempvar misscount dropmarker drop_anyways_marker
    **## determine variable list to check
    local dropped_main_ids
    local drop_anyways_count 0
    local checkvars : list varlist - idvar
    local checkvarcount : word count `checkvars'
    local plural=cond(`checkvarcount'>1,"s","")
    local info_addon=cond(${casi_popro_debug}," in dataset {input:`c(filename)'}","")
    _print_info checking for observations without valid information for {input:`checkvarcount'} variable`plural'`info_addon'
    _print_debug {input:`checkvars'}
    local ignorevalues_comma : subinstr local ignorevalues " " "," , all
    **## remove completely missing observations
    quietly {
        generate `misscount' = 0
        foreach checkvar of local checkvars {
            if ( usubstr(`"`: type `checkvar''"',1,3) == "str" ) {
                quietly: replace `misscount'=`misscount'+1 if ( inlist(`checkvar',`macval(ignorestrings)') | missing(`checkvar') )
            }
            else {
                quietly: replace `misscount'=`misscount'+1 if ( inlist(`checkvar',`macval(ignorevalues_comma)') | missing(`checkvar') )
            }
        }
        generate `dropmarker' = ( `misscount' == `checkvarcount' )
        generate `drop_anyways_marker' = 0
        count if ( `dropmarker' == 1 )
        local dropped `r(N)'
        local plural=cond(`dropped'>1,"s","")
        **### check if no observations left in dataset after dropping, keep all if so -- unless some ids are on the `drop_anyways' list
        if (`dropped' == `c(N)') {
            if ( !missing(`"`drop_anyways'"') ) {
                foreach entry of local drop_anyways {
                    if ( usubstr(`"`: type `idvar''"',1,3) == "str" ) {
                        local check_condition `"`idvar' == `"`macval(entry)'"'"'
                    }
                    else {
                        local check_condition `"`idvar' == `macval(entry)'"'
                    }
                    replace `drop_anyways_marker' = 1 if ( `check_condition' )
                }
                count if ( `drop_anyways_marker' == 1 & `dropmarker' == 1 )
                local drop_anyways_count = `r(N)'
            }
            local plural=cond(`dropped'>1,"s","")
            _print_warning there are no observations with valid values in the current dataset; but if we dropped all {result:`dropped'} observation`plural', none would be left in the dataset and merging the data would fail; so we're not dropping observations without valid data for now
            if ( `drop_anyways_count' == 0 | `drop_anyways_count' == `c(N)' ) {
                return scalar dropped = 0
                exit 0
            }
            local dropped : copy local drop_anyways_count
            local plural=cond(`dropped'>1,"s","")
            local have=cond(`dropped'>1,"have","has")
            _print_warning we will, however, drop {result:`dropped'} observation`plural' that `have' been dropped from one or more previous source dataset(s) for the sake of consistency
            replace `dropmarker' = 0 if ( `drop_anyways_marker' != 1 )
        }
        if (`dropped' > 0) {
            local dropcondition `"`dropmarker' == 1"'
            levelsof `idvar' if ( `dropcondition' ) , local(dropped_main_ids)
            capture: confirm variable lastpage, exact
            if (_rc==0) {
                local dropped_main_idpairs
                foreach entry of local dropped_main_ids {
                    if ( usubstr(`"`: type `idvar''"',1,3)==`"str"' ) {
                        levelsof lastpage if ( `idvar'==`"`entry'"' ) , local(lastpage_content) clean
                    }
                    else {
                        levelsof lastpage if ( `idvar'==`entry' ) , local(lastpage_content) clean
                    }
                    local new_entry `"`""`entry'":"`lastpage_content'""'"'
                    local dropped_main_idpairs: list dropped_main_idpairs | new_entry
                }
                local dropped_main_ids: copy local dropped_main_idpairs
            }
            drop if ( `dropcondition' )
        }
    }
    if ( `dropped' > 0 ) {
        _print_info dropped {result:`dropped'} observation`plural' without valid values
    }
    return scalar dropped = `dropped'
    return local dropped_main_ids = `"`dropped_main_ids'"'
    exit 0
end

program define _check_for_empty_dataset, rclass
    **## check if this is an empty dataset, or one without non-missing observations for the id variable, skip if yes
    syntax anything(name=filepath id="file name"), idvar(string)
    preserve
    use `"`filepath'"', clear
    local skip_me 0
    if ( `c(N)'==0 ) {
        _print_warning Dataset {input:`filepath'} does not contain any observations and will be skipped!
        local skip_me 1
    }
    capture : confirm variable `idvar', exact
    if ( `c(rc)'!=0 ) {
        _print_error Main id variable {input:`idvar'} not found in dataset {input:`filepath'}! Cannot continue.
        exit 111
    }
    quietly : count if missing(`idvar')
    if ( `r(N)'==`c(N)' ) {
        _print_warning Dataset {input:`filepath'} does not contain any observations with non-missing observations for {input:`idvar'} and will be skipped!
        local skip_me 1
    }
    return local skip_me = `skip_me'
    restore
    exit 0
end

program define _prettify_limesurvey_techvars, nclass
	local varlabels `""submitdate:survey fully concluded at (timestamp)" "startdate:survey started at  (timestamp)" "datestamp:last respondent activity at  (timestamp)" "lastpage:last survey page for respondent""'
    foreach pair of local varlabels {
		gettoken var rest : pair, parse(":")
        capture: confirm variable `var', exact
        if (_rc!=0) {
            exit 0
        }
    }
	tempvar drop_var
	foreach pair of local varlabels {
		gettoken var rest : pair, parse(":")
		gettoken colon varlabel: rest, parse(":")
		assert `"`colon'"'==":"
		if inlist(`"`var'"',"submitdate","startdate","datestamp") {
			rename `var' `drop_var'
			quietly: generate double `var'=clock(`drop_var',"YMD hms") if ( !missing(`drop_var') ), after(`drop_var')
			format `var' %tcCCYY-NN-DD_HH:MM:SS
			drop `drop_var'
		}
		label variable `var' `"`varlabel'"'
	}
	exit 0
end

program define _process_network_survey, nclass
    syntax , ///
        selected_value(numlist integer max=1) /// the value to be replaced with 'idtarget'
        unselected_value(numlist integer max=1) /// the value that should be in the variable when it is *not* selected
        varlist_clusters(string)              // the quoted (!) variable lists to be processed; each quoted group will be processed separately
    local id_detection_problems 0
    **## process variables lists
    foreach cluster_varlist of local varlist_clusters {
        local namevarlist
        **### check variable presence
        foreach namevar of local cluster_varlist {
            capture: confirm variable `namevar', exact
            if (_rc==0) {
                local namevarlist: list namevarlist | namevar
            }
        }
        if missing(`"`namevarlist'"') {
            _print_debug … no variable matching to naming variable list {input:`cluster_varlist'} found, cannot process this list
            continue
        }
        _print_info … processing variable list with {result:`: word count `namevarlist''} out of {input:`: word count `cluster_varlist''} possible variables
        _print_debug {result:`namevarlist'}
        **### replace "selected" values with idtargets
        foreach namevar of local namevarlist {
            local namevarlabel: variable label `namevar'
            assert inlist(`namevar',0,1) | inrange(`namevar',-99,-20)
            if ustrregexm(`"`namevarlabel'"',`"idtarget ([0-9]+)"') {
                local focus_idtarget=ustrregexs(1)
            }
            else if ustrregexm(`"`namevarlabel'"',`".* Kind ([0-9]+)"') {
                local id_detection_problems 1
                local id_detection_sample_label: copy local namevarlabel
                local focus_idtarget=ustrregexs(1)
            }
            else {
                _print_error … no {it:idtarget} found from child focus variable label {input:`namevarlabel'} for variable {input:`namevar'}, skipping variable
                continue
            }
            _print_debug … replacing content: variable {input:`namevar'} `selected_value' --> `focus_idtarget'
                quietly: replace `namevar'=`focus_idtarget' if (`namevar'==`selected_value')
        }
        if ( `id_detection_problems' ) {
            _print_warning … no {it:idtarget} found from one or more child focus variable labels, but running child numbers!
            _print_warning … example variable label: {input:`id_detection_sample_label'}

        }
        c_local id_detection_problems `id_detection_problems'
        **### move idtargets to the left in case of empty values (==non-selected)
        tempvar fillin_marker
        local varcount 0
        foreach namevar of local namevarlist {
            local ++ varcount
            local nextcount=`varcount'+1
            if `nextcount' > `: word count `namevarlist'' {
                continue
            }
            foreach nextvar in `ferest()' {
                _print_debug …filling gap data: `namevar' <-- `nextvar'
                quietly {
                    generate `fillin_marker'=(`nextvar'!=`unselected_value' & `nextvar'>0 & !missing(`nextvar') & `namevar'==`unselected_value')
                    replace `namevar'=`nextvar' if (`fillin_marker')
                    replace `nextvar'=0 if (`fillin_marker')
                }
                drop `fillin_marker'
            }
        }
    }
    exit 0
end

program define _usage_msg, nclass
noisily: display _newline as result in smcl "casi_popro.ado: Stata post-processor for NEPS CASI raw data"_newline(2) ///
    "Usage:"_newline as text"{tab}{bf:casi_popro} , {opt studynumber(name)} [ {it:options} ]"_newline(2) ///
    `"{tab}(with {it:name} as the NEPS (sub) study name to insert into the result dataset file names)"'_newline(2) ///
    `"{synoptset 25 tabbed}{...}"'_newline ///
    `"{synopthdr}"'_newline ///
    `"{synoptline}"'_newline(2) ///
    `"{syntab:File paths options:}"'_newline ///
    `"{synopt:{opt id_list(filename [, keepsource replace nofakeids])}}use {it:filename} as a lookup table for inserting IDs to the final dataset; valid file types for {it:filename} are old-format Microsoft Excel files ({input:.xls}), OOXML ({input:.xlsx}), CSV ({input:.csv}), tab-separated ({input:.tsv}), or Stata dataset ({input:.dta}) files);{p_end}"'_newline ///
    `"{synopt:}if the sub-option {it:keepsource} is specified, do not erase the original ID variable;{p_end}"'_newline ///
    `"{synopt:}if the sub-option {it:replace} is specified, replace the target ID variable if it is already existing;{p_end}"'_newline ///
    `"{synopt:}if the sub-option {it:nofakeids} is specified, do not generate fake ids in case of non-matched observations in the id list (but keep them as system missing values){p_end}"'_newline ///
    `"{synopt:{opt inputdirectory(dirname)}}use {it:dirname} as the source directory to read ZIP files from (defaults to {input:./input} if omitted){p_end}"'_newline ///
    `"{synopt:{opt outputdirectory(dirname)}}use {it:dirname} as the output directory to save files to (defaults to {input:./output} if omitted){p_end}"'_newline ///
    `"{synopt:{opt temp_unpack_path(dirname)}}use {it:dirname} as the path to temporarily extract all survey and assessment data (defaults to a random directory inside of {input:`c(tmpdir)'} if omitted); {bf:if specified, this directory will be auto-created at the beginning and erased without warning at the end of the script, unless the option {opt nocleanup} is specified!}{p_end}"'_newline ///
    `"{synopt:{opt temp_address_unpack_path(dirname)}}use {it:dirname} as the path to temporarily extract all address data (defaults to a random directory inside of {input:`c(tmpdir)'} if omitted); {bf:if specified, this directory will be auto-created at the beginning and erased without warning at the end of the script, unless the option {opt nocleanup} is specified!}{p_end}"'_newline(2) ///
    `"{syntab:File naming options:}"'_newline ///
    `"{synopt:{opt version(version_string)}}append {it:version_string} to the result dataset file names{p_end}"'_newline ///
    `"{synopt:{opt focusperson(shorthand)}}insert {it:shorthand} as focus person denominator into the result dataset file names (defaults to {input:T} if omitted; valid values are: {input:T}, {input:P}, {input:H}, {input:E}, {input:I}, {input:C}){p_end}"'_newline ///
    `"{synopt:{opt replace}}if result dataset files are already existent, replace them{p_end}"'_newline(2) ///
    `"{syntab:Options for episode data processing:}"'_newline ///
    `"{synopt:{opt spellvarname(varname)}}use {it:varname} as spell counter in episode data (defaults to {it:spell} if omitted){p_end}"'_newline ///
    `"{synopt:{opt spell_unused_value(value)}}assume {it:value} as indicator for unused spell loop variables (defaults to {it:-99} if omitted){p_end}"'_newline ///
    `"{synopt:{opt keep_empty_spells}}if specified, include spells without valid values in episode result datasets (default is to drop such spells){p_end}"'_newline ///
    `"{synopt:{opt reshape_threshold(number)}}when detecting episode data, only trigger processing when {it:number} or more wide variables per loop are detected (defaults to {input:2} if omitted){p_end}"'_newline ///
    `"{synopt:{opt join_threshold(number)}}when matching episode data, only combine data sources if they have {it:number} or more variables in common (defaults to {input:2} if omitted){p_end}"'_newline(2) ///
    `"{syntab:Expert options:}"'_newline ///
    `"{synopt:{opt id_variable(varlist)}}use {varlist} as valid ID variables; the first variable in {it:varlist} will be used as main ID variable in the result data, and all others will be renamed to it accordingly (defaults to {it:token index} if omitted){p_end}"'_newline ///
    `"{synopt:{opt dropvars(varlist)}}drop {namelist} from result data before saving (defaults to {it:id seed startlanguage OUTnavigate*} if omitted){p_end}"'_newline ///
    `"{synopt:{opt ignore_duplicate_vars(varlist)}}ignore {varlist} when trying to auto-resolve duplicate observations (defaults to {it:{c 'g}dropvars' submitdate lastpage startdate datestamp} if omitted){p_end}"'_newline ///
    `"{synopt:{opt ignore_duplicate_varlabelregex(regex)}}ignore all variables whose variable label matches {it:regex} when trying to auto-resolve duplicate observations (defaults to {it:.*\[(AUTO)|(HILF)\].*} if omitted){p_end}"'_newline ///
    `"{synopt:{opt nodropofemptycases}}if specified, do not drop empty observations (i.e. observations without valid data in any variable) from datasets (default is to drop such observations){p_end}"'_newline ///
    `"{synopt:{opt noexporttimestamp}}if specified, do not save export time stamp and host name to the result data (default is to save variables {result:casi_export_host} and {result:casi_export_time}){p_end}"'_newline ///
    `"{synopt:{opt cawimode}}if specified, only processing steps applicable for CAWI exports are performed{p_end}"'_newline ///
    `"{synopt:{opt nocleanup}}if specified, do not remove temporary files after saving result data{p_end}"'_newline ///
    `"{synopt:{opt debug}}if specified, give verbose output during processing{p_end}"'_newline ///
    `"{p2colreset}{...}"'
    exit 0
end
**# EOF