text:
/* $Id: parse_opts.c,v 1.1 2004/07/21 09:10:41 rb878c Exp $ */

/**********************************************************
 *
 *    OS Testing - Silicon Graphics, Inc.
 *
 *    FUNCTION NAME 	: parse_opts
 *
 *    FUNCTION TITLE	: parse standard & user options for system call tests
 *
 *    SYNOPSIS:
 *	#include "usctest.h"
 *
 * 	char *parse_opts(ac, av, user_optarr, uhf)
 *	int    ac;
 * 	char **av;
 *	option_t user_optarr[];
 *	void (*uhf)();
 *
 *    AUTHOR		: William Roske/Richard Logan
 *
 *    INITIAL RELEASE	: UNICOS 7.0
 *
 *    DESCRIPTION
 * 	The parse_opts library routine takes that argc and argv parameters
 *	recevied by main() and an array of structures defining user options.
 *	It parses the command line setting flag and argument locations
 *      associated with the options.  It uses getopt to do the actual cmd line
 *      parsing.  uhf() is a function to print user define help
 *
 *      This module contains the functions usc_global_setup_hook and
 *      usc_test_looping, which are called by marcos defined in usctest.h.
 *
 *    RETURN VALUE
 * 	parse_opts returns a pointer to an error message if an error occurs.
 *	This pointer is (char *)NULL if parsing is successful.
 *
 *#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#*#**/
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sys/param.h>
#include <sys/signal.h>
#include <sys/types.h>
#include <unistd.h>
#include <sys/time.h>

#if UNIT_TEST
#include <time.h>
#endif /* UNIT_TEST */

#include "test.h"
#define _USC_LIB_   1	/* indicates we are the library to the usctest.h include */
#include "usctest.h"

#ifndef USC_COPIES
#define USC_COPIES   "USC_COPIES"
#endif

#ifndef UNIT_TEST
#define UNIT_TEST	0
#endif

#ifndef DEBUG
#define DEBUG	0
#endif

/* The timing information block. */
struct tblock tblock={0,((long) -1)>>1,0,0};

/* Define flags and args for standard options */
int STD_FUNCTIONAL_TEST=1,	/* flag indicating to do functional testing code */
    STD_TIMING_ON=0,		/* flag indicating to print timing stats */
    STD_PAUSE=0,		/* flag indicating to pause before actual start, */
				/* for contention mode */
    STD_INFINITE=0,		/* flag indciating to loop forever */
    STD_LOOP_COUNT=1,		/* number of iterations */
    STD_COPIES=1,		/* number of copies */
    STD_ERRNO_LOG=0;		/* flag indicating to do errno logging */

float STD_LOOP_DURATION=0.0,    /* duration value in fractional seconds */
      STD_LOOP_DELAY=0.0;	/* loop delay value in fractional seconds */

char **STD_opt_arr = NULL;	/* array of option strings */
int    STD_nopts=0,		/* number of elements in STD_opt_arr */
       STD_argind=1;		/* argv index to next argv element */
				/* (first argument) */
				/* To getopt users, it is like optind */

/*
 * The following variables are to support system testing additions.
 */
static int  STD_TP_barrier=0;	/* flag to do barrier in TEST_PAUSE */
				/* 2 - wait_barrier(), 3 - set_barrier(), * - barrier() */
static int  STD_LP_barrier=0;	/* flag to do barrier in TEST_LOOPING */
				/* 2 - wait_barrier(), 3 - set_barrier(), * - barrier() */
static int  STD_TP_shmem_sz=0;	/* shmalloc this many words per pe in TEST_PAUSE */
static int  STD_LD_shmem=0; 	/* flag to do shmem_puts and shmem_gets during delay */
static int  STD_LP_shmem=0; 	/* flag to do shmem_puts and gets during TEST_LOOPING */
static int  STD_LD_recfun=0;	/* do recressive function calls in loop delay */
static int  STD_LP_recfun=0;	/* do recressive function calls in TEST_LOOPING */
static int  STD_TP_sbrk=0;	/* do sbrk in TEST_PAUSE */
static int  STD_LP_sbrk=0;	/* do sbrk in TEST_LOOPING */
static char *STD_start_break=0; /* original sbrk size */
static int  Debug=0;

struct std_option_t {
    char *optstr;
    char *help;
    char *flag;
    char **arg;
} std_options[] = {
    { "c:", "  -c n    Run n copies concurrently\n", NULL, NULL},
    { "e" , "  -e      Turn on errno logging\n", NULL, NULL},
    { "f" , "  -f      Turn off functional testing\n", NULL, NULL},
    { "h" , "  -h      Show this help screen\n", NULL, NULL},
    { "i:", "  -i n    Execute test n times\n", NULL, NULL},
    { "I:", "  -I x    Execute test for x seconds\n", NULL, NULL},
    { "p" , "  -p      Pause for SIGUSR1 before starting\n", NULL, NULL},
    { "P:", "  -P x    Pause for x seconds between iterations\n", NULL, NULL},
    { "t" , "  -t      Turn on syscall timing\n", NULL, NULL},
    {NULL, NULL, NULL, NULL}};

void print_help(void (*user_help)());

/*
 * Structure for usc_recressive_func argument
 */
struct usc_bigstack_t {
   char space[4096];
};

static struct usc_bigstack_t *STD_bigstack=NULL;

/*
 * Counter of errnos returned (-e option).  Indexed by errno.
 * Make the array USC_MAX_ERRNO long.  That is the first Fortran
 * Lib errno.  No syscall should return an errno that high.
 */
int STD_ERRNO_LIST[USC_MAX_ERRNO];

/* define the string length for Mesg and Mesg2 strings */
#define STRLEN 2048

static char Mesg2[STRLEN];	/* holds possible return string */
static void usc_recressive_func();

/*
 * Define bits for options that might have env variable default
 */
#define  OPT_iteration 		01
#define  OPT_nofunccheck	02
#define  OPT_duration		04
#define  OPT_delay		010
#define  OPT_copies		020

/**********************************************************************
 * parse_opts:
 **********************************************************************/
char *
parse_opts(int ac, char **av, option_t *user_optarr, void (*uhf)())
{
    int found;		/* flag to indicate that an option specified was */
			/* found in the user's list */
    int k;		/* scratch integer for returns and short time usage */
    float  ftmp;	/* tmp float for parsing env variables */
    char *ptr;		/* used in getting env variables */
    int options=0;	/* no options specified */
    int optstrlen, i;
    char *optionstr;
    int opt;		/* return of getopt */

    /*
     * If not the first time this function is called, release the old STD_opt_arr
     * vector.
     */

    if ( STD_opt_arr != NULL ) {
	free(STD_opt_arr);
	STD_opt_arr=NULL;
    }
    /* Calculate how much space we need for the option string */
    optstrlen = 0;
    for (i = 0; std_options[i].optstr; ++i)
	optstrlen += strlen(std_options[i].optstr);
    if (user_optarr)
	for (i = 0; user_optarr[i].option; ++i) {
	    if (strlen(user_optarr[i].option) > 2)
		return "parse_opts: ERROR - Only short options are allowed";
	    optstrlen += strlen(user_optarr[i].option);
	}
    optstrlen += 1;

    /* Create the option string for getopt */
    optionstr = (char *)malloc(optstrlen);
    if (!optionstr)
	return "parse_opts: ERROR - Could not allocate memory for optionstr";

    optionstr[0] = '\0';

    for (i = 0; std_options[i].optstr; ++i)
	strcat(optionstr, std_options[i].optstr);
    if (user_optarr)
	for (i = 0; user_optarr[i].option; ++i)
	    /* only add the option if it wasn't there already */
	    if (strchr(optionstr, user_optarr[i].option[0]) == NULL)
		strcat(optionstr, user_optarr[i].option);

#if DEBUG > 1
    printf("STD_nopts = %d\n", STD_nopts);
#endif

    /*
     *  Loop through av parsing options.
     */
    while ( (opt = getopt(ac, av, optionstr)) > 0) {

	STD_argind = optind;
#if DEBUG > 0
	printf("parse_opts: getopt returned '%c'\n", opt);
#endif

	switch (opt) {
		case '?': /* Unknown option */
			return "Unknown option";
			break;
		case ':': /* Missing Arg */
			return "Missing argument";
			break;
		case 'i': /* Iterations */
			options |= OPT_iteration;
			STD_LOOP_COUNT = atoi(optarg);
			if (STD_LOOP_COUNT == 0) STD_INFINITE = 1;
			break;
		case 'P': /* Delay between iterations */
			options |= OPT_delay;
			STD_LOOP_DELAY = atof(optarg);
			break;
		case 'I': /* Time duration */
			options |= OPT_duration;
			STD_LOOP_DURATION = atof(optarg);
			if ( STD_LOOP_DURATION == 0.0 ) STD_INFINITE=1;
			break;
		case 'c': /* Copies */
			options |= OPT_copies;
			STD_COPIES = atoi(optarg);
			break;
		case 'f': /* Functional testing */
			STD_FUNCTIONAL_TEST = 0;
			break;
		case 'p': /* Pause for SIGUSR1 */
			STD_PAUSE = 1;
			break;
		case 't': /* syscall timing */
			STD_TIMING_ON = 1;
			break;
		case 'e': /* errno loggin */
			STD_ERRNO_LOG = 1;
			break;
		case 'h': /* Help */
			print_help(uhf);
			exit(0);
			break;
		default:

            /* Check all the user specified options */
            found=0;
	    for(i = 0; user_optarr[i].option; ++i) {

		if (opt == user_optarr[i].option[0]) {
                    /* Yup, This is a user option, set the flag and look for argument */
		    if ( user_optarr[i].flag ) {
                        *user_optarr[i].flag=1;
		    }
                    found++;

		    /* save the argument at the user's location */
                    if ( user_optarr[i].option[strlen(user_optarr[i].option)-1] == ':' ) {
                        *user_optarr[i].arg=optarg;
                    }
                    break;  /* option found - break out of the for loop */
                }
            }
	    /* This condition "should never happen".  SO CHECK FOR IT!!!! */
            if ( ! found ) {
                sprintf(Mesg2,
		    "parse_opts: ERROR - option:\"%c\" NOT FOUND... INTERNAL ERROR", opt);
                return(Mesg2);
            }
	}

    }    /* end of while */
    free(optionstr);

    STD_argind = optind;

    /*
     * Turn on debug
     */
    if ( (ptr=getenv("USC_DEBUG")) != NULL ) {
	Debug=1;
        printf("env USC_DEBUG is defined, turning on debug\n");
    }
    if ( (ptr=getenv("USC_VERBOSE")) != NULL ) {
	Debug=1;
        printf("env USC_VERBOSE is defined, turning on debug\n");
    }

    /*
     * If the USC_ITERATION_ENV environmental variable is set to
     * a number, use that number as iteration count (same as -c option).
     * The -c option with arg will be used even if this env var is set.
     */
    if ( !(options & OPT_iteration) && (ptr=getenv(USC_ITERATION_ENV)) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1) {
            if ( k == 0 ) {   /* if arg is 0, set infinite loop flag */
                STD_INFINITE=1;
		if ( Debug )
		   printf("Using env %s, set STD_INFINITE to 1\n",
			USC_ITERATION_ENV);
            } else {            /* else, set the loop count to the arguement */
                STD_LOOP_COUNT=k;
		if ( Debug )
		   printf("Using env %s, set STD_LOOP_COUNT to %d\n",
			USC_ITERATION_ENV, k);
            }
        }
    }

    /*
     * If the USC_NO_FUNC_CHECK environmental variable is set, we'll
     * unset the STD_FUNCTIONAL_TEST variable.
     */
    if ( !(options & OPT_nofunccheck) && (ptr=getenv(USC_NO_FUNC_CHECK)) != NULL ) {
        STD_FUNCTIONAL_TEST=0; /* Turn off functional testing */
	if ( Debug )
	    printf("Using env %s, set STD_FUNCTIONAL_TEST to 0\n",
		USC_NO_FUNC_CHECK);
    }

    /*
     * If the USC_LOOP_WALLTIME environmental variable is set,
     * use that number as duration (same as -I option).
     * The -I option with arg will be used even if this env var is set.
     */

    if ( !(options & OPT_duration) && (ptr=getenv(USC_LOOP_WALLTIME)) != NULL ) {
        if ( sscanf(ptr, "%f", &ftmp) == 1 && ftmp >= 0.0 ) {
	    STD_LOOP_DURATION=ftmp;
	    if ( Debug )
	        printf("Using env %s, set STD_LOOP_DURATION to %f\n",
		    USC_LOOP_WALLTIME, ftmp);
            if ( STD_LOOP_DURATION == 0.0 ) {   /* if arg is 0, set infinite loop flag */
                STD_INFINITE=1;
	        if ( Debug )
	            printf("Using env %s, set STD_INFINITE to 1\n", USC_LOOP_WALLTIME);
	    }
        }
    }
    if ( !(options & OPT_duration) && (ptr=getenv("USC_DURATION")) != NULL ) {
        if ( sscanf(ptr, "%f", &ftmp) == 1 && ftmp >= 0.0 ) {
	    STD_LOOP_DURATION=ftmp;
	    if ( Debug )
	        printf("Using env USC_DURATION, set STD_LOOP_DURATION to %f\n", ftmp);
            if ( STD_LOOP_DURATION == 0.0 ) {   /* if arg is 0, set infinite loop flag */
                STD_INFINITE=1;
	        if ( Debug )
	            printf("Using env USC_DURATION, set STD_INFINITE to 1\n");
	    }
        }
    }
    /*
     * If the USC_LOOP_DELAY environmental variable is set,
     * use that number as delay in factional seconds (same as -P option).
     * The -P option with arg will be used even if this env var is set.
     */
    if ( !(options & OPT_delay) && (ptr=getenv(USC_LOOP_DELAY)) != NULL ) {
        if ( sscanf(ptr, "%f", &ftmp) == 1 && ftmp >= 0.0 ) {
	    STD_LOOP_DELAY=ftmp;
	    if ( Debug )
		printf("Using env %s, set STD_LOOP_DELAY = %f\n",
		    USC_LOOP_DELAY, ftmp);
        }
    }

    /*
     * If the USC_COPIES environmental variable is set,
     * use that number as copies (same as -c option).
     * The -c option with arg will be used even if this env var is set.
     */
    if ( !(options & OPT_copies) && (ptr=getenv(USC_COPIES)) != NULL ) {
        if ( sscanf(ptr, "%d", &STD_COPIES) == 1 && STD_COPIES >= 0 ) {
	    if ( Debug )
		printf("Using env %s, set STD_COPIES = %d\n",
		    USC_COPIES, STD_COPIES);
        }
    }

    /*
     * The following are special system testing envs to turn on special
     * hooks in the code.
     */
    if ( (ptr=getenv("USC_TP_BARRIER")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
	    STD_TP_barrier=k;
	}
        else
	    STD_TP_barrier=1;
	if ( Debug )
	    printf("using env USC_TP_BARRIER, Set STD_TP_barrier to %d\n",
	        STD_TP_barrier);
    }

    if ( (ptr=getenv("USC_LP_BARRIER")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
	    STD_LP_barrier=k;
	}
        else
	    STD_LP_barrier=1;
	if ( Debug )
	    printf("using env USC_LP_BARRIER, Set STD_LP_barrier to %d\n",
	        STD_LP_barrier);
    }

    if ( (ptr=getenv("USC_TP_SHMEM")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
            STD_TP_shmem_sz=k;
	    if ( Debug )
	        printf("Using env USC_TP_SHMEM, Set STD_TP_shmem_sz to %d\n",
		    STD_TP_shmem_sz);
        }
    }

    if ( (ptr=getenv("USC_LP_SHMEM")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
            STD_LP_shmem=k;
	    if ( Debug )
	        printf("Using env USC_LP_SHMEM, Set STD_LP_shmem to %d\n",
		    STD_LP_shmem);
        }
    }

    if ( (ptr=getenv("USC_LD_SHMEM")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
            STD_LD_shmem=k;
	    if ( Debug )
	        printf("Using env USC_LD_SHMEM, Set STD_LD_shmem to %d\n",
		    STD_LD_shmem);
        }
    }

    if ( (ptr=getenv("USC_TP_SBRK")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
            STD_TP_sbrk=k;
	    if ( Debug )
	        printf("Using env USC_TP_SBRK, Set STD_TP_sbrk to %d\n",
		    STD_TP_sbrk);
        }
    }

    if ( (ptr=getenv("USC_LP_SBRK")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
            STD_LP_sbrk=k;
	    if ( Debug )
	        printf("Using env USC_LP_SBRK, Set STD_LP_sbrk to %d\n",
		    STD_LP_sbrk);
        }
    }

    if ( (ptr=getenv("USC_LP_RECFUN")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
	    STD_LP_recfun=k;
	    if ( STD_bigstack != (struct usc_bigstack_t *)NULL )
		STD_bigstack=(struct usc_bigstack_t *)
			malloc(sizeof(struct usc_bigstack_t));
	    if ( Debug )
                printf("Using env USC_LP_RECFUN, Set STD_LP_recfun to %d\n",
		    STD_LP_recfun);
        }
    }

    if ( (ptr=getenv("USC_LD_RECFUN")) != NULL ) {
        if ( sscanf(ptr, "%i", &k) == 1 && k >= 0 ) {
	    STD_LD_recfun=k;
	    if ( STD_bigstack != (struct usc_bigstack_t *)NULL )
		STD_bigstack=(struct usc_bigstack_t *)
			malloc(sizeof(struct usc_bigstack_t));
	    if ( Debug )
                printf("Using env USC_LD_RECFUN, Set STD_LD_recfun to %d\n",
		    STD_LD_recfun);
        }
    }

#if UNIT_TEST
    printf("The following variables after option and env parsing:\n");
    printf("STD_FUNCTIONAL_TEST = %d\n", STD_FUNCTIONAL_TEST);
    printf("STD_LOOP_DURATION   = %f\n", STD_LOOP_DURATION);
    printf("STD_LOOP_DELAY      = %f\n", STD_LOOP_DELAY);
    printf("STD_COPIES          = %d\n", STD_COPIES);
    printf("STD_LOOP_COUNT      = %d\n", STD_LOOP_COUNT);
    printf("STD_INFINITE        = %d\n", STD_INFINITE);
    printf("STD_TIMING_ON       = %d\n", STD_TIMING_ON);
    printf("STD_ERRNO_LOG       = %d\n", STD_ERRNO_LOG);
    printf("STD_PAUSE           = %d\n", STD_PAUSE);
#endif

    return((char *) NULL);

}    /* end of parse_opts */

/*********************************************************************
 * print_help() - print help message and user help message
 *********************************************************************/
void print_help(void (*user_help)())
{
    STD_opts_help();

    if (user_help) user_help();
}

/*********************************************************************
 * STD_opts_help() - return a help string for the STD_OPTIONS.
 *********************************************************************/
void
STD_opts_help()
{
    int i;

    for(i = 0; std_options[i].optstr; ++i) {
	if (std_options[i].help)
	    printf(std_options[i].help);
    }
}

/*
 * routine to goto when we get the SIGUSR1 for STD_PAUSE
 */
void STD_go(int sig)
{
   return;
}

/***********************************************************************
 * This function will do desired end of global setup test
 * hooks.
 * Currently it will only do a pause waiting for sigusr1 if
 * STD_PAUSE is set.
 *
 ***********************************************************************/
int
usc_global_setup_hook()
{
    int cnt;
    /* temp variable to store old signal action to be restored after pause */
    int (*_TMP_FUNC)(void);

    /*
     * Fork STD_COPIES-1 copies.
     */
    for(cnt=1;cnt<STD_COPIES;cnt++) {
        switch(fork() ) {
	    case -1:
		fprintf(stderr, "%s: fork() failed, errno:%d %s\n",
                __FILE__, errno, strerror(errno));
		break;
	    case 0:  /* child */
	        cnt=STD_COPIES;   /* to stop the forking */
		break;

	    default: /* parent */
		break;
	}
    }

    /*
     * pause waiting for sigusr1.
     */
    if ( STD_PAUSE ) {
        _TMP_FUNC = (int (*)())signal(SIGUSR1, STD_go);
        pause();
        signal(SIGUSR1, (void (*)())_TMP_FUNC);
    }

    if ( STD_TP_sbrk || STD_LP_sbrk) {
	STD_start_break=sbrk(0);	/* get original sbreak size */
    }

    if ( STD_TP_sbrk ) {
	sbrk(STD_TP_sbrk);
	if ( Debug )
	    printf("after sbrk(%d)\n", STD_TP_sbrk);
    }
    return 0;
}

#define USECS_PER_SEC	1000000  /* microseconds per second */

/***********************************************************************
 * This function returns the number of get_current_time()'s return
 * per second.
 ***********************************************************************/

static int
get_timepersec()
{
    return  USECS_PER_SEC;   /* microseconds per second */

}

/***********************************************************************
 * this function will get current time in microseconds since 1970.
 ***********************************************************************/
static int
get_current_time()
{
    struct timeval curtime;

    gettimeofday(&curtime, NULL);

    /* microseconds since 1970 */
    return (curtime.tv_sec*USECS_PER_SEC) + curtime.tv_usec;

}

/***********************************************************************
 *
 * This function will determine if test should continue iterating
 * If the STD_INFINITE flag is set, return 1.
 * If the STD_LOOP_COUNT variable is set, compare it against
 * the counter.
 * If the STD_LOOP_DURATION variable is set, compare current time against
 * calculated stop_time.
 * This function will return 1 until all desired looping methods
 * have been met.
 *
 * counter integer is supplied by the user program.
 ***********************************************************************/
int
usc_test_looping(counter)
int counter;
{
    static int first_time = 1;
    static int stop_time = 0;	/* stop time in rtc or usecs */
    static int delay;		/* delay in clocks or usecs  */
    int hz=0;			/* clocks per second or usecs per second */
    int ct, end;		/* current time, end delay time */
    int keepgoing=0;		/* used to determine return value */

    /*
     * If this is the first iteration and we are looping for
     * duration of STD_LOOP_DURATION seconds (fractional) or
     * doing loop delays, get the clocks per second.
     */
    if ( first_time ) {

	first_time=0;
	if ( STD_LOOP_DELAY || STD_LOOP_DURATION ) {
	    hz = get_timepersec();
	}

	/*
	 * If looping for duration, calculate stop time in
	 * clocks.
	 */

	if ( STD_LOOP_DURATION) {
	    ct=get_current_time();
	    stop_time=(int)((float)hz * STD_LOOP_DURATION) + ct;
	}

	/*
	 * If doing delay each iteration, calcuate the number
	 * of clocks for each delay.
	 */
	if ( STD_LOOP_DELAY ) {
	    delay=(int)((float)hz * STD_LOOP_DELAY);
	}

    }

    /*
     * if delay each iteration, loop for delay clocks.
     * This will not be done on first iteration.
     * The delay will happen before determining if
     * there will be another iteration.
     */
    else if ( STD_LOOP_DELAY ) {
	ct=get_current_time();
        end=ct+delay;
        while ( ct < end ) {
	    /*
	     * The following are special test hooks in the delay loop.
	     */
	    if ( STD_LD_recfun ) {
		if ( Debug )
		    printf("calling usc_recressive_func(0, %d, &STD_bigstack)\n",
		        STD_LD_recfun);
		usc_recressive_func(0, STD_LD_recfun, &STD_bigstack);
	    }

	    ct=get_current_time();
	}
    }

    if ( STD_INFINITE ) {
	keepgoing++;
    }

    if ( STD_LOOP_COUNT && counter < STD_LOOP_COUNT ) {
	keepgoing++;
    }

    if ( STD_LOOP_DURATION != 0.0 && get_current_time() < stop_time ) {
	keepgoing++;
    }

    if ( keepgoing == 0 )
	return 0;

    /*
     * The following code allows special system testing hooks.
     */

    if ( STD_LP_recfun ) {
	if ( Debug )
	    printf("calling usc_recressive_func(0, %d, &STD_bigstack)\n",
	        STD_LP_recfun);
	usc_recressive_func(0, STD_LP_recfun, &STD_bigstack);
    }

    if ( STD_LP_sbrk ) {
	if ( Debug )
	    printf("about to do sbrk(%d)\n", STD_LP_sbrk);
	sbrk(STD_LP_sbrk);
    }

    if ( keepgoing )
	return 1;
    else
        return 0;	/* done - stop iterating */
}

/*
 * This function recressively calls itself max times.
 */
static void
usc_recressive_func(cnt, max, bstack)
int cnt;
int max;
struct usc_bigstack_t bstack;
{
    if ( cnt < max )
	usc_recressive_func(cnt+1, max, bstack);

}

#if UNIT_TEST
/******************************************************************************
 * UNIT TEST CODE
 * UNIT TEST CODE
 *
 * this following code is provide so that unit testing can
 * be done fairly easily.
 ******************************************************************************/

int Help = 0;
int Help2 = 0;
char *ptr;

/*
 * Code from usctest.h that not part of this file since we are the library.
 */

struct usc_errno_t TEST_VALID_ENO[USC_MAX_ERRNO];

  /***********************************************************************
   * Globals for returning the return code and errno from the system call
   * test macros.
   ***********************************************************************/
int TEST_RETURN;
int TEST_ERRNO;

  /***********************************************************************
   * temporary variables for determining max and min times in TEST macro
   ***********************************************************************/
long btime, etime, tmptime;

/* for test specific parse_opts options */
option_t Options[] = {
        { "help",  &Help2, NULL },      /* -help option */
        { "h",  &Help, NULL },          /* -h option */
 	{ TIMING, NULL, NULL},		/* disable -timing option */

#if INVALID_TEST_CASES
 	{ "missingflag", NULL, &ptr },  /* error */
 	{ "missingarg:", &Help, NULL },  /* error */
#endif  /* INVALID_TEST_CASES */

        { NULL, NULL, NULL }
};

main(argc, argv)
int argc;
char **argv;
{
    int lc;
    char *msg;
    struct timeval t;
    int cnt;

    if ( (msg=parse_opts(argc, argv,
			(option_t *) Options)) != (char *) NULL ) {
	printf("ERROR : %s\n", msg);
	exit(1);
    }

    TEST_PAUSE;

    for (lc=0; TEST_LOOPING(lc); lc++) {

        TEST( gettimeofday(&t, NULL) );
        printf("iter=%d: sec:%d, usec:%6.6d %s", lc+1, t.tv_sec,
	    t.tv_usec, ctime(&t.tv_sec));
    }

    TEST_CLEANUP;

    exit(0);
}

#endif /* UNIT_TEST */


text:
import { EventEmitter } from "events";
import { AccentConverter } from "./AccentConverter";
import { DictionaryFinder } from "./DictionaryFinder";
import { ResourceManager } from "./ResourceManager";
import { registerResourceManagers } from "./DictionaryRegister";
import { IWordDefinition } from "./model/IWordDefinition";
import { Sqlite } from "./util/Sqlite";
import { IgnorableTrie } from "./IgnorableTrie";
import { HTMLCreator } from "./HTMLCreator";
import { IIndex } from "./model/IIndex";
import { IDictParserOptions } from "./model/IDictParserOptions";
import { OSSpecificImplementationGetter } from "./os-specific/OSSpecificImplementationGetter";

export class DictParser extends EventEmitter {
  private _sqliteDbPath: string;
  private _wordFormsFolder: string;
  private _dictionaryFinder = new DictionaryFinder();
  private _dictionaries = this._dictionaryFinder.dictionaries;
  private _vocabulary: IgnorableTrie;

  public constructor(options: IDictParserOptions) {
    super();
    OSSpecificImplementationGetter.path = options.pathImplementation;
    OSSpecificImplementationGetter.fs = options.fsImplementation;
    OSSpecificImplementationGetter.sqlite = options.sqliteImplementation;
    OSSpecificImplementationGetter.os = options.os;
    ResourceManager.commonResourceDirectory = options.commonResourceDirectory;
    this._wordFormsFolder = options.wordFormsFolder;
    this._sqliteDbPath = options.sqliteDbPath;
    registerResourceManagers();
  }
  public init = async (): Promise<void> => {
    // this._dictMapList = await this.loadDictMapList();
    await Sqlite.init(this._sqliteDbPath);
    await this.loadVocabulary();
  };

  public async scan(scanFolder: string | string[]): Promise<void> {
    await Sqlite.reset();
    this._dictionaryFinder.on("name", (dictionaryName: string) => {
      this.emit("name", dictionaryName);
    });
    await this._dictionaryFinder.scan(Array.isArray(scanFolder) ? scanFolder : [scanFolder]);
  }

  /**
   * Guess what word the user wants based on the word input
   */
  public async getWordCandidates(input: string, resultCount = 30): Promise<string[]> {
    // input = this.normalize(input);

    if (input.trim() === "") {
      return [];
    }

    const candidates = this._vocabulary.findWordsStartWith(input, resultCount);
    this._vocabulary.findWordsStartWithExcludeIgnoreCharacters(input, resultCount, candidates);
    return Array.from(candidates);
  }

  public async getWordDefinitions(word: string): Promise<IWordDefinition[]> {
    const wordDefinitionList: IWordDefinition[] = [];
    const wordIndexList = await this.queryWordIndexes(word);
    for (const wordIndex of wordIndexList) {
      const dictionary = this._dictionaries.get(wordIndex.dictionary.type);
      if (dictionary === undefined) {
        continue;
      }
      try {
        const html = await dictionary.getDefinition(wordIndex.dictionary, wordIndex.word, wordIndex.pos, wordIndex.len);
        wordDefinitionList.push({ word, html, dict: wordIndex.dictionary });
      } catch (e) {
        console.error(e);
        wordDefinitionList.push({
          word,
          html: HTMLCreator.getFailedToFetchDefinitionFromDictionaryHTML(word, wordIndex.dictionary.name),
          dict: wordIndex.dictionary
        });
      }
    }
    return wordDefinitionList;
  }

  public loadVocabulary = async (): Promise<void> => {
    const vocabulary = new IgnorableTrie();
    const queryResultList = await Sqlite.db.getAll("SELECT DISTINCT word FROM word_index");
    for (const queryResult of queryResultList) {
      vocabulary.add(queryResult.word);
    }
    this._vocabulary = vocabulary;
  };

  private normalize(word: string): string {
    let normalizedWord: string = "";
    for (const c of word) {
      normalizedWord += AccentConverter.removeAccent(c).toLowerCase();
    }
    return normalizedWord.trim();
  }

  // private loadAllWordsFromIndexMap = (
  //   indexMap: IndexMap,
  //   vocabulary: Trie
  // ) => {
  //   for (const word in indexMap) {
  //     if (indexMap.hasOwnProperty(word)) {
  //       vocabulary.add(word);
  //     }
  //   }
  // };
  private queryWordIndexes = async (word: string): Promise<IIndex[]> => {
    const queryResultList = await Sqlite.db.getAll(
      `
      SELECT
        word_index.id                 AS id,
        word_index.word               AS word,
        word_index.pos                AS pos,
        word_index.len                AS len,
        dictionary.id                 AS dictionary_id,
        dictionary.name               AS name,
        dictionary.word_count         AS word_count,
        dictionary.syn_path           AS syn_path,
        dictionary.index_path         AS index_path,
        dictionary.resource_path      AS resource_path,
        dictionary.dict_path          AS dict_path,
        dictionary.ann_path           AS ann_path,
        dictionary.bmp_path           AS bmp_path,
        dictionary.same_type_sequence AS same_type_sequence,
        dictionary.type               AS type
      FROM word_index INNER JOIN dictionary
        ON word_index.dictionary_id = dictionary.id
      WHERE word = ?
    `,
      [word]
    );
    const wordIndexList: IIndex[] = [];
    for (const queryResult of queryResultList) {
      wordIndexList.push({
        id: queryResult.id,
        dictionary: {
          id: parseInt(queryResult.dictionary_id, 10),
          name: queryResult.name,
          wordCount: parseInt(queryResult.word_count, 10),
          synPath: queryResult.syn_path,
          indexPath: queryResult.index_path,
          resourcePath: queryResult.resource_path,
          dictPath: queryResult.dict_path,
          annPath: queryResult.ann_path,
          bmpPath: queryResult.bmp_path,
          sameTypeSequence: queryResult.same_type_sequence,
          type: queryResult.type
        },
        word: queryResult.word,
        len: parseInt(queryResult.len, 10),
        pos: parseInt(queryResult.pos, 10)
      });
    }
    return wordIndexList;
  };
  // private addSimilarWords = (
  //   input: string,
  //   vocabulary: Trie,
  //   resultCount: number,
  //   result: Set<string>
  // ) => {
  //   for (const word of Array.from(vocabulary.values())) {
  //     if (word.startsWith(input)) {
  //       result.add(word);
  //       if (result.size > resultCount) {
  //         return;
  //       }
  //     }
  //   }
  // };
}

redpajama_set_name: RedPajamaGithub


text:
package com.microsoft.azure.cognitiveservices.vision.computervision.models;

import com.fasterxml.jackson.annotation.JsonProperty;

/**
 * An image caption, i.e. a brief description of what the image depicts.
 */
public class ImageCaption {
    /**
     * The text of the caption.
     */
    @JsonProperty(value = "text")
    private String text;

    /**
     * The level of confidence the service has in the caption.
     */
    @JsonProperty(value = "confidence")
    private double confidence;

    /**
     * Get the text value.
     *
     * @return the text value
     */
    public String text() {
        return this.text;
    }

    /**
     * Set the text value.
     *
     * @param text the text value to set
     * @return the ImageCaption object itself.
     */
    public ImageCaption withText(String text) {
        this.text = text;
        return this;
    }

    /**
     * Get the confidence value.
     *
     * @return the confidence value
     */
    public double confidence() {
        return this.confidence;
    }

    /**
     * Set the confidence value.
     *
     * @param confidence the confidence value to set
     * @return the ImageCaption object itself.
     */
    public ImageCaption withConfidence(double confidence) {
        this.confidence = confidence;
        return this;
    }

}


text:
package components;

import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;

import components.entityComponents.BackgroundComponent;
import components.entityComponents.ComponentType;

public abstract class AbstractStringComponent extends AComponent implements IComponent {
	private String myString;

	public AbstractStringComponent(String inputString) {
		myString = inputString;
	}

	public AbstractStringComponent() {
	}

	public String getString() {
		return myString;
	}

	public void setString(String inputString) {
		myString = inputString;
	}

	@Override
	public IComponent newCopy() {
		Class c = this.getClass();
		Constructor ctor;
		if (myString==null){
			try {
				ctor = c.getConstructor();
				return (IComponent) ctor.newInstance();
			} catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
				//add Error Handline here;
			}

		}else {
			try {
				ctor = c.getConstructor(String.class);
				return (IComponent) ctor.newInstance(myString);
			} catch (NoSuchMethodException| SecurityException | IllegalArgumentException |InstantiationException | IllegalAccessException | InvocationTargetException e) {
			}
		}
		return null;
	}
}


text:
<?xml version="1.0" encoding="utf-8"?>
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:id="@+id/list_widget_item"
    android:layout_width="match_parent"
    android:layout_height="match_parent">
  <TextView xmlns:android="http://schemas.android.com/apk/res/android"
    android:id="@+id/list_widget_item_text"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:layout_margin="10dp"
    android:textColor="#ffffff"
    android:textSize="40px" />
</FrameLayout>


text:
namespace chromeos {

namespace device_sync {

FakeDeviceSyncObserver::FakeDeviceSyncObserver() = default;

FakeDeviceSyncObserver::~FakeDeviceSyncObserver() = default;

mojo::PendingRemote<mojom::DeviceSyncObserver>
FakeDeviceSyncObserver::GenerateRemote() {
  mojo::PendingRemote<mojom::DeviceSyncObserver> remote;
  receivers_.Add(this, remote.InitWithNewPipeAndPassReceiver());
  return remote;
}

void FakeDeviceSyncObserver::OnEnrollmentFinished() {
  ++num_enrollment_events_;
}

void FakeDeviceSyncObserver::OnNewDevicesSynced() {
  ++num_sync_events_;
}

}  // namespace device_sync

}  // namespace chromeos


text:
'use strict';

// Requires
var firebase = require('firebase');
var request = require('request');
var mkdirp = require('mkdirp');
var path = require('path');
var fs = require('fs');
var glob = require('glob');
var tinylr = require('tiny-lr');
var _ = require('lodash');
var wrench = require('wrench');
var utils = require('./utils.js');
var ws = require('ws').Server;
var Zip   = require('adm-zip');
var slug = require('uslug');
var async = require('async');
var spawn = require('win-spawn');
var md5 = require('MD5');
var $ = require('cheerio');
var exec = require('child_process').exec;

require('colors');

// Template requires
// TODO: Abstract these later to make it simpler to change
var swig = require('swig');
swig.setDefaults({ loader: swig.loaders.fs(__dirname + '/..') });

var swigFunctions = require('./swig_functions').swigFunctions();
var swigFilters = require('./swig_filters');
var swigTags = require('./swig_tags');
swigFilters.init(swig);
swigTags.init(swig);
swig.setDefaults({ cache: false });

var wrap = function()
{
  var args = Array.prototype.slice.call(arguments);

  var last = args.pop();
  last = 'debugger;' +
         'var global = null;' +
         'var console = null;' +
         'var v8debug = null;' +
         'var setTimeout = null;' +
         'var setInterval = null;' +
         'var setImmediate = null;' +
         'var clearTimeout = null;' +
         'var clearInterval = null;' +
         'var clearImmediate = null;' +
         'var root = null;' +
         'var GLOBAL = null;' +
         'var window = null;' +
         'var process = null;' +
         'var eval = null;' +
         last;

  args.push(last);

  return Function.prototype.constructor.apply(this, args);
};
wrap.prototype = Function.prototype;
Function = wrap;

// Disable console log in various things
//console.log = function () {};

/**
 * Generator that handles various commands
 * @param  {Object}   config     Configuration options from .firebase.conf
 * @param  {Object}   logger     Object to use for logging, defaults to no-ops
 */
module.exports.generator = function (config, options, logger, fileParser) {
  var self = this;
  var firebaseUrl = config.get('webhook').firebase || 'webhook';
  var liveReloadPort = config.get('connect')['wh-server'].options.livereload;
  var websocket = null;
  var strictMode = false;
  var productionFlag = false;

  this.versionString = null;
  this.cachedData = null;

  if(liveReloadPort === true)
  {
    liveReloadPort = 35729;
  }

  logger = logger || { ok: function() {}, error: function() {}, write: function() {}, writeln: function() {} };

  // We dont error out here so init can still be run
  if (firebaseUrl)
  {
    this.root = new firebase('https://' + firebaseUrl +  '.firebaseio.com/');
  } else {
    this.root = null;
  }

  /**
   * Used to get the bucket were using (combinaton of config and environment)
   */
  var getBucket = function() {
    return self.root.child('buckets/' + config.get('webhook').siteName + '/' + config.get('webhook').secretKey + '/dev');
  };

  /**
   * Used to get the dns information about a site (used for certain swig functions)
   */
  var getDnsChild = function() {
    return self.root.child('management/sites/' + config.get('webhook').siteName + '/dns');
  };

  var getTypeData = function(type, callback) {
    getBucket().child('contentType').child(type).once('value', function(data) {
      callback(data.val());
    });
  }

  /**
   * Retrieves snapshot of data from Firebase
   * @param  {Function}   callback   Callback function to run after data is retrieved, is sent the snapshot
   */
  var getData = function(callback) {

    if(self.cachedData)
    {
      swigFunctions.setData(self.cachedData.data);
      swigFunctions.setTypeInfo(self.cachedData.typeInfo);
      swigFunctions.setSettings(self.cachedData.settings);
      swigFilters.setSiteDns(self.cachedData.siteDns);
      swigFilters.setFirebaseConf(config.get('webhook'));

      callback(self.cachedData.data, self.cachedData.typeInfo);
      return;
    }

    if(!self.root)
    {
      throw new Error('Missing firebase reference, may need to run init');
    }

    getBucket().once('value', function(data) {
      data = data.val();
      var typeInfo = {};
      var settings = {};

      if(!data || !data['contentType'])
      {
        typeInfo = {};
      } else {
        typeInfo = data['contentType'];
      }

      if(!data || !data.settings) {
        settings = {};
      } else {
        settings = data.settings;
      }

      // Get the data portion of bucket, other things are not needed for templates
      if(!data || !data.data) {
        data = {};
      } else {
        data = data.data;
      }

      self.cachedData = {
        data: data,
        typeInfo: typeInfo,
        settings: settings
      };

      // Sets the context for swig functions
      swigFunctions.setData(data);
      swigFunctions.setTypeInfo(typeInfo);
      swigFunctions.setSettings(settings);

      getDnsChild().once('value', function(snap) {
        var siteDns = snap.val() || config.get('webhook').siteName + '.webhook.org';
        self.cachedData.siteDns = siteDns;
        swigFilters.setSiteDns(siteDns);
        swigFilters.setFirebaseConf(config.get('webhook'));

        callback(data, typeInfo);
      });
    }, function(error) {
      if(error.code === 'PERMISSION_DENIED') {
        console.log('\n========================================================'.red);
        console.log('# Permission denied                                         #'.red);
        console.log('========================================================'.red);
        console.log('#'.red + ' You don\'t have permission to this site or your subscription expired.'.red);
        console.log('# Visit '.red + 'https://billing.webhook.com/site/'.yellow + config.get('webhook').siteName.yellow + '/'.yellow  + ' to manage your subscription.'.red);
        console.log('# ---------------------------------------------------- #'.red)
        process.exit(0);
      } else {
        throw new Error(error);
      }
    });
  };

  var searchEntryStream = null;

  this.openSearchEntryStream = function(callback) {
    if(config.get('webhook').noSearch === true) {
      callback();
      return;
    }

    if(!fs.existsSync('./.build/.wh/')) {
      mkdirp.sync('./.build/.wh/');
    }

    searchEntryStream = fs.createWriteStream('./.build/.wh/searchjson.js');

    searchEntryStream.write('var tipuesearch = {"pages": [\n');

    callback();
  };

  this.closeSearchEntryStream = function(callback) {
    if(config.get('webhook').noSearch === true || !searchEntryStream) {
      callback();
      return;
    }

    searchEntryStream.end(']}');

    searchEntryStream.on('close', function() {
      callback();
    });
  };

  var writeSearchEntry = function(outFile, output) {
    if(config.get('webhook').noSearch === true || !searchEntryStream) {
      return;
    }

    var endUrl = outFile.replace('./.build', '');

    if(path.extname(endUrl) !== '.html' || endUrl === '/404.html' || endUrl.indexOf('/_wh_previews') === 0) {
      return;
    }

    endUrl = endUrl.replace('index.html', '');

    var content = $.load(output);

    var title = content('title').text();
    var bodyObj = content('body');

    if(bodyObj.attr('data-search-index') === "false") {
      return;
    }

    var specialChild = bodyObj.find('[data-search-index="true"]');

    if(specialChild.length > 0) {
      bodyObj = specialChild.first();
    }

    bodyObj.find('script').remove();
    bodyObj.find('iframe').remove();
    bodyObj.find('object').remove();
    bodyObj.find('[data-search-index="false"]').remove();

    var body = bodyObj.text().trim();
    var tags = "";

    if(content('meta[name="keywords"]').length > 0) {
      tags = content('meta[name="keywords"]').attr('content');
    }

    if(searchEntryStream) {
      var searchObj = {
        title: title,
        text: body,
        tags: tags,
        loc: endUrl
      };

      searchEntryStream.write(JSON.stringify(searchObj) + ',\n');

      searchObj = null;
    }

    title = '';
    body = '';
  };

  /**
   * Writes an instance of a template to the build directory
   * @param  {string}   inFile     Template to read
   * @param  {string}   outFile    Destination in build directory
   * @param  {Object}   params     The parameters to pass to the template
   */
  var writeTemplate = function(inFile, outFile, params) {
    params = params || {};
    params['firebase_conf'] = config.get('webhook');
    var originalOutFile = outFile;

    // Merge functions in
    params = utils.extend(params, swigFunctions.getFunctions());

    swigFunctions.init();

    var outputUrl = outFile.replace('index.html', '').replace('./.build', '');
    swigFunctions.setParams({ CURRENT_URL: outputUrl });

    if(params.item) {
      params.item = params._realGetItem(params.item._type, params.item._id, true);
    }

    params.production = productionFlag;

    var output = '';
    try {
      output = swig.renderFile(inFile, params);
    } catch (e) {
      self.sendSockMessage(e.toString());

      if(strictMode) {
        throw e;
      } else {
        console.log('Error while rendering template: ' + inFile);
        console.log(e.toString().red);
        try {
          output = swig.renderFile('./libs/debug500.html', { template: inFile, error: e.toString(), backtrace: e.stack.toString() })
        } catch (e) {
          return '';
        }
      }
    }

    mkdirp.sync(path.dirname(outFile));
    fs.writeFileSync(outFile, output);
    writeSearchEntry(outFile, output);

    // Haha this crazy nonsense is to handle pagination, the swig function "paginate" makes
    // shouldPaginate return true if there are more pages left, so we enter a while loop to
    // generate each page of the pagination (todo one day, abstract this with above code into simple functions)
    swigFunctions.increasePage();
    while(swigFunctions.shouldPaginate())
    {
      outFile = originalOutFile.replace('/index.html', '/' + swigFunctions.pageUrl + swigFunctions.curPage + '/index.html');
      outputUrl = outFile.replace('index.html', '').replace('./.build', '');

      swigFunctions.setParams({ CURRENT_URL: outputUrl });

      try {
        var output = swig.renderFile(inFile, params);
      } catch (e) {
        self.sendSockMessage(e.toString());

        if(strictMode) {
          throw e;
        } else {
          console.log('Error while rendering template: ' + inFile);
          console.log(e.toString().red);
          try {
            output = swig.renderFile('./libs/debug500.html', { template: inFile, error: e.toString(), backtrace: e.stack.toString() })
          } catch (e) {
            return '';
          }
        }
      }

      mkdirp.sync(path.dirname(outFile));
      fs.writeFileSync(outFile, output);
      writeSearchEntry(outFile, output);

      swigFunctions.increasePage();
    }

    return outFile.replace('./.build', '');
  };

  /**
   * Downloads a zip file from the requested url and extracts it into the main directory
   * @param  {string}   zipUrl     Url to zip file to download
   * @param  {Function}   callback   Callback, first parameter is error (true if error occured);
   */
  var downloadRepo = function(zipUrl, callback) {
    logger.ok('Downloading preset...');

    // Keep track if the request fails to prevent the continuation of the install
    var requestFailed = false;

    // TODO: have this hit different templating repos
    var repoRequest = request(zipUrl);

    repoRequest
    .on('response', function (response) {
      // If we fail, set it as failing and remove zip file
      if (response.statusCode !== 200) {
        requestFailed = true;
        fs.unlinkSync('.preset.zip');
        callback(true);
      }
    })
    .pipe(fs.createWriteStream('.preset.zip'))
    .on('close', function () {
      if (requestFailed) return;

      // Unzip into temporary file
      var zip = new Zip('.preset.zip');

      var entries = zip.getEntries();

      entries.forEach(function(entry) {
        var newName = entry.entryName.split('/').slice(1).join('/');
        entry.entryName = newName;
      });
      zip.extractAllTo('.', true);

      fs.unlinkSync('.preset.zip');
      callback();
    });
  };

  var resetGenerator = function(callback) {
    logger.ok('Resetting Generator...');
    var zipUrl = 'http://dump.webhook.com/static/generate-repo.zip';

    // Keep track if the request fails to prevent the continuation of the install
    var requestFailed = false;

    // TODO: have this hit different templating repos
    var repoRequest = request(zipUrl);

    repoRequest
    .on('response', function (response) {
      // If we fail, set it as failing and remove zip file
      if (response.statusCode !== 200) {
        requestFailed = true;
        fs.unlinkSync('.reset.zip');
        callback(true);
      }
    })
    .pipe(fs.createWriteStream('.reset.zip'))
    .on('close', function () {
      if (requestFailed) return;

      // Unzip into temporary file
      var zip = new Zip('.reset.zip');

      var entries = zip.getEntries();

      removeDirectory('.pages-old', function() {
        removeDirectory('.templates-old', function() {
          removeDirectory('.static-old', function() {

            try {
              fs.renameSync('pages', '.pages-old');
            } catch(error) {
              fs.unlinkSync('.reset.zip');
              callback(true);
              return;
            }

            try {
              fs.renameSync('templates', '.templates-old');
            } catch(error) {
              fs.renameSync('.pages-old', 'pages');
              fs.unlinkSync('.reset.zip');
              callback(true);
              return;
            }

            try {
              fs.renameSync('static', '.static-old');
            } catch(error) {
              fs.renameSync('.pages-old', 'pages');
              fs.renameSync('.templates-old', 'templates');
              fs.unlinkSync('.reset.zip');
              callback(true);
              return;
            }

            entries.forEach(function(entry) {
              if(entry.entryName.indexOf('pages/') === 0
                 || entry.entryName.indexOf('templates/') === 0
                 || entry.entryName.indexOf('static/') === 0) {
                zip.extractEntryTo(entry.entryName, '.', true, true);
              }
            });

            removeDirectory('.pages-old', function() {
              removeDirectory('.templates-old', function() {
                removeDirectory('.static-old', function() {
                  fs.unlinkSync('.reset.zip');

                  self.init(config.get('webhook').siteName, config.get('webhook').secretKey, true, config.get('webhook').firebase, function() {
                    callback();
                  });
                });
              });
            });

          });
        });
      });
    });
  };

  /**
  * Extracts a local theme zip into the current generator directory
  * @param zipUrl   The location of the zip file on disk
  * @param callback The callback to call with the data from the theme
  */
  var extractPresetLocal = function(fileData, callback) {

    fs.writeFileSync('.preset.zip', fileData, { encoding: 'base64' });
    // Unzip into temporary file
    var zip = new Zip('.preset.zip');

    var entries = zip.getEntries();

    entries.forEach(function(entry) {
      var newName = entry.entryName.split('/').slice(1).join('/');
      entry.entryName = newName;
    });
    zip.extractAllTo('.', true);

    fs.unlinkSync('.preset.zip');

    if(fs.existsSync('.preset-data.json')) {
      var presetData = fileParser.readJSON('.preset-data.json');

      fs.unlinkSync('.preset-data.json');
      logger.ok('Done extracting.');
      callback(presetData);

    } else {
      logger.ok('Done extracting.');
      callback(null);
    }
  }

  /**
   * Downloads zip file and then sends the preset data for the theme to the CMS for installation
   * @param  {string}   zipUrl     Url to zip file to download
   * @param  {Function}   callback   Callback, first parameter is preset data to send to CMS
   */
  var downloadPreset = function(zipUrl, callback) {
    downloadRepo(zipUrl, function() {
      if(fs.existsSync('.preset-data.json')) {
        var presetData = fileParser.readJSON('.preset-data.json');

        fs.unlinkSync('.preset-data.json');
        logger.ok('Done downloading.');
        callback(presetData);

      } else {
        logger.ok('Done downloading.');
        callback(null);
      }
    });
  };

  /**
   * Renders all templates in the /pages directory to the build directory
   * @param  {Function}   done     Callback passed either a true value to indicate its done, or an error
   * @param  {Function}   cb       Callback called after finished, passed list of files changed and done function
   */
  this.renderPages = function (done, cb)  {
    logger.ok('Rendering Pages\n');

    getData(function(data) {

      glob('pages/**/*', function(err, files) {
        files.forEach(function(file) {

          if(fs.lstatSync(file).isDirectory()) {
            return true;
          }

          var newFile = file.replace('pages', './.build');

          var dir = path.dirname(newFile);
          var filename = path.basename(newFile, path.extname(file));
          var extension = path.extname(file);

          if(path.extname(file) === '.html' && filename !== 'index' && path.basename(newFile) !== '404.html') {
            dir = dir + '/' + filename;
            filename = 'index';
          }

          newFile = dir + '/' + filename + path.extname(file);

          if(extension === '.html' || extension === '.xml' || extension === '.rss' || extension === '.xhtml' || extension === '.atom' || extension === '.txt') {
            writeTemplate(file, newFile);
          } else {
            mkdirp.sync(path.dirname(newFile));
            fs.writeFileSync(newFile, fs.readFileSync(file));
          }
        });

        if(fs.existsSync('./libs/.supported.js')) {
          mkdirp.sync('./.build/.wh/_supported');
          fs.writeFileSync('./.build/.wh/_supported/index.html', fs.readFileSync('./libs/.supported.js'));
        }

        logger.ok('Finished Rendering Pages\n');

        if(cb) cb(done);
      });

    });
  };

  var generatedSlugs = {};
  var generateSlug = function(value) {
    if(!generatedSlugs[value._type]) {
      generatedSlugs[value._type] = {};
    }

    if(value.slug) {
      generatedSlugs[value._type][value.slug] = true;
      return value.slug;
    }
    var tmpSlug = slug(value.name).toLowerCase();

    var no = 2;
    while(generatedSlugs[value._type][tmpSlug]) {
      tmpSlug = slug(value.name).toLowerCase() + '_' + no;
      no++;
    }

    generatedSlugs[value._type][tmpSlug] = true;

    return tmpSlug;
  }

  /**
   * Renders all templates in the /templates directory to the build directory
   * @param  {Function}   done     Callback passed either a true value to indicate its done, or an error
   * @param  {Function}   cb       Callback called after finished, passed list of files changed and done function
   */
  this.renderTemplates = function(done, cb) {
    logger.ok('Rendering Templates');
    generatedSlugs = {};

    getData(function(data, typeInfo) {

      glob('templates/**/*.html', function(err, files) {

        files.forEach(function(file) {
          // We ignore partials, special directory to allow making of partial includes
          if(path.extname(file) === '.html' && file.indexOf('templates/partials') !== 0)
          {
            if(path.dirname(file).split('/').length <= 1) {
              return true;
            }
            // Here we try and abstract out the content type name from directory structure
            var baseName = path.basename(file, '.html');
            var newPath = path.dirname(file).replace('templates', './.build').split('/').slice(0,3).join('/');

            var pathParts = path.dirname(file).split('/');
            var objectName = pathParts[1];
            var items = data[objectName];
            var info = typeInfo[objectName];
            var filePath = path.dirname(file);
            var overrideFile = null;

            if(!items) {
              logger.error('Missing data for content type ' + objectName);
            }

            items = _.map(items, function(value, key) { value._id = key; value._type = objectName; return value });

            var publishedItems = _.filter(items, function(item) {
              if(!item.publish_date) {
                return false;
              }

              var now = Date.now();
              var pdate = Date.parse(item.publish_date);

              if(pdate > now + (1 * 60 * 1000)) {
                return false;
              }

              return true;
            });

            var baseNewPath = '';

            // Find if this thing has a template control
            var templateWidgetName = null;

            if(typeInfo[objectName]) {
              typeInfo[objectName].controls.forEach(function(control) {
                if(control.controlType === 'layout') {
                  templateWidgetName = control.name;
                }
              });
            }

            var listPath = null;

            if(typeInfo[objectName] && typeInfo[objectName].customUrls && typeInfo[objectName].customUrls.listUrl) {
              var customPathParts = newPath.split('/');

              if(typeInfo[objectName].customUrls.listUrl === '#') // Special remove syntax
              {
                listPath = customPathParts.join('/');
                customPathParts.splice(2, 1);
              } else {
                customPathParts[2] = typeInfo[objectName].customUrls.listUrl;
              }

              newPath = customPathParts.join('/');
            }

            var origNewPath = newPath;

            // TODO, DETECT IF FILE ALREADY EXISTS, IF IT DOES APPEND A NUMBER TO IT DUMMY
            if(baseName === 'list')
            {
              newPath = newPath + '/index.html';

              if(listPath) {
                newPath = listPath + '/index.html';
              }

              writeTemplate(file, newPath);

            } else if (baseName === 'individual') {
              // Output should be path + id + '/index.html'
              // Should pass in object as 'item'
              baseNewPath = newPath;
              var previewPath = baseNewPath.replace('./.build', './.build/_wh_previews');

              // TODO: Check to make sure file does not exist yet, and then adjust slug if it does? (how to handle in swig functions)
              for(var key in publishedItems)
              {
                var val = publishedItems[key];

                if(templateWidgetName && val[templateWidgetName]) {
                  overrideFile = 'templates/' + objectName + '/layouts/' + val[templateWidgetName];
                }

                var addSlug = true;
                if(val.slug) {
                  baseNewPath = './.build/' + val.slug + '/';
                  addSlug = false;
                } else {
                  if(typeInfo[objectName] && typeInfo[objectName].customUrls && typeInfo[objectName].customUrls.individualUrl) {
                    baseNewPath = origNewPath + '/' + utils.parseCustomUrl(typeInfo[objectName].customUrls.individualUrl, val) + '/';
                  } else {
                    baseNewPath = origNewPath + '/';
                  }
                }

                var tmpSlug = '';
                if(!val.slug) {
                  tmpSlug = generateSlug(val);
                } else {
                  tmpSlug = val.slug;
                }

                if(addSlug) {
                  val.slug = baseNewPath.replace('./.build/', '') + tmpSlug;
                  newPath = baseNewPath + tmpSlug + '/index.html';
                } else {
                  newPath = baseNewPath + 'index.html';
                }

                if(fs.existsSync(overrideFile)) {
                  writeTemplate(overrideFile, newPath, { item: val });
                  overrideFile = null;
                } else {
                  writeTemplate(file, newPath, { item: val });
                }
              }

              for(var key in items)
              {
                var val = items[key];

                if(templateWidgetName && val[templateWidgetName]) {
                  overrideFile = 'templates/' + objectName + '/layouts/' + val[templateWidgetName];
                }

                newPath = previewPath + '/' + val.preview_url + '/index.html';

                if(fs.existsSync(overrideFile)) {
                  writeTemplate(overrideFile, newPath, { item: val });
                  overrideFile = null;
                } else {
                  writeTemplate(file, newPath, { item: val });
                }
              }
            } else if(filePath.indexOf('templates/' + objectName + '/layouts') !== 0) { // Handle sub pages in here
              baseNewPath = newPath;

              var middlePathName = filePath.replace('templates/' + objectName, '') + '/' + baseName;
              middlePathName = middlePathName.substring(1);

              for(var key in publishedItems)
              {
                var val = publishedItems[key];

                var addSlug = true;
                if(val.slug) {
                  baseNewPath = './.build/' + val.slug + '/';
                  addSlug = false;
                } else {
                  if(typeInfo[objectName] && typeInfo[objectName].customUrls && typeInfo[objectName].customUrls.individualUrl) {
                    baseNewPath = origNewPath + '/' + utils.parseCustomUrl(typeInfo[objectName].customUrls.individualUrl, val) + '/';
                  }   else {
                    baseNewPath = origNewPath + '/';
                  }
                }

                var tmpSlug = '';
                if(!val.slug) {
                  tmpSlug = generateSlug(val);
                } else {
                  tmpSlug = val.slug;
                }

                if(addSlug) {
                  val.slug = baseNewPath.replace('./.build/', '') + tmpSlug;
                  newPath = baseNewPath + tmpSlug + '/' + middlePathName + '/index.html';
                } else {
                  newPath = baseNewPath + middlePathName + '/index.html';
                }

                writeTemplate(file, newPath, { item: val });
              }
            }
          }
        });

        logger.ok('Finished Rendering Templates');

        if(cb) cb(done);

      });
    });
  };

  /**
   * Copies the static directory into .build/static for asset generation
   * @param  {Function}   callback     Callback called after creation of directory is done
   */
  this.copyStatic = function(callback) {
    logger.ok('Copying static');
    if(fs.existsSync('static')) {
      mkdirp.sync('.build/static');
      wrench.copyDirSyncRecursive('static', '.build/static', { forceDelete: true });
    }
    callback();
  };

  var removeDirectory = function(directory, callback) {
    var isWin = /^win/.test(process.platform);

    if(isWin) {
      exec('rmdir /s /q ' + directory, function(err) {

        if(err) {
          if(fs.existsSync(directory)) {
            wrench.rmdirSyncRecursive(directory);
          }
        }
        callback();
      });
    } else {
      if(fs.existsSync(directory)) {
        wrench.rmdirSyncRecursive(directory);
      }

      callback();
    }
  }

  /**
   * Cleans the build directory
   * @param  {Function}   done     Callback passed either a true value to indicate its done, or an error
   * @param  {Function}   cb       Callback called after finished, passed list of files changed and done function
   */
  this.cleanFiles = function(done, cb) {
      logger.ok('Cleaning files');

      removeDirectory('.build', function() {
        if (cb) cb();
        if (done) done(true);
      });
  };

  /**
   * Builds templates from both /pages and /templates to the build directory
   * @param  {Function}   done     Callback passed either a true value to indicate its done, or an error
   * @param  {Function}   cb       Callback called after finished, passed list of files changed and done function
   */
  this.buildBoth = function(done, cb) {
    // clean files
    self.cachedData = null;
    self.cleanFiles(null, function() {
      self.openSearchEntryStream(function() {
        self.renderTemplates(null, function() {
          self.copyStatic(function() {
            self.renderPages(function() {}, function() {
              self.closeSearchEntryStream(function() {
                cb(done);
              });
            });
          });
        });
      })
    });
  };

  this.checkScaffoldingMD5 = function(name, callback) {
    self.cachedData = null;
    getData(function(data, typeInfo) {
      var directory = 'templates/' + name + '/';
      var individual = directory + 'individual.html';
      var list = directory + 'list.html';
      var oneOff = 'pages/' + name + '.html';

      var individualMD5 = null;
      var listMD5 = null;
      var oneOffMD5 = null;

      if(typeInfo[name].oneOff) {
        if(fs.existsSync(oneOff)) {
          var oneOffContent = fs.readFileSync(oneOff);
          oneOffMD5 = md5(oneOffContent);
        }
      } else {
        if(fs.existsSync(individual)) {
          var indContent = fs.readFileSync(individual);
          individualMD5 = md5(indContent);
        }

        if(fs.existsSync(list)) {
          var listContent = fs.readFileSync(list);
          listMD5 = md5(listContent);
        }
      }

      callback(individualMD5, listMD5, oneOffMD5);
    });
  }

  /**
   * Generates scaffolding for content type with name
   * @param  {String}   name     Name of content type to generate scaffolding for
   * @param  {Function}   done     Callback called when scaffolding generation is done
   * @param  {Boolean}   force    If true, forcibly overwrites old scaffolding
   */
  this.makeScaffolding = function(name, done, force) {
    logger.ok('Creating Scaffolding for ' + name + '\n');
    var directory = 'templates/' + name + '/';

    var list = directory + 'list.html';
    var individual = directory +  'individual.html';
    var oneOff = 'pages/' + name + '.html';

    var individualTemplate = fs.readFileSync('./libs/scaffolding_individual.html');
    var listTemplate = fs.readFileSync('./libs/scaffolding_list.html');
    var oneOffTemplate = fs.readFileSync('./libs/scaffolding_oneoff.html');

    var widgetFilesRaw = [];

    if(fs.existsSync('./libs/widgets')) {
      widgetFilesRaw = wrench.readdirSyncRecursive('./libs/widgets');
    }

    var widgetFiles = [];

    widgetFilesRaw.forEach(function(item) {
      widgetFiles[(path.dirname(item) + '/' + path.basename(item, '.html')).replace('./', '')] = true;
    });

    var renderWidget = function(controlType, fieldName, controlInfo, overridePrefix) {
      var controls = [];

      if(controlInfo.controls) {
        _.each(controlInfo.controls, function(item) {
          controls[item.name] = item;
        });
      }

      var prefix = overridePrefix || 'item.';

      var widgetString = _.template(fs.readFileSync('./libs/widgets/' + controlType + '.html'), { value: prefix + fieldName, controlInfo: controlInfo, renderWidget: renderWidget, controls: controls, widgetFiles: widgetFiles });

      var lines = widgetString.split('\n');
      var newLines = [];
      var first = true;

      lines.forEach(function(line) {
        if(first) {
          first = false;
          newLines.push(line);
        } else {
          var newLine = '        ' + line;
          newLines.push(newLine);
        }
      });

      return newLines.join('\n');
    };

    self.cachedData = null;
    getData(function(data, typeInfo) {
      var controls = typeInfo[name] ? typeInfo[name].controls : [];
      var controlsObj = {};

      _.each(controls, function(item) {
        controlsObj[item.name] = item;
      });

      var individualMD5 = null;
      var listMD5 = null;
      var oneOffMD5 = null;

      if(typeInfo[name].oneOff) {
        if(!force && fs.existsSync(oneOff)) {
          if(done) done(null, null, null);
          logger.error('Scaffolding for ' + name + ' already exists, use --force to overwrite');
          return false;
        }

        var oneOffFile = _.template(oneOffTemplate, { widgetFiles: widgetFiles, typeName: name, typeInfo: typeInfo[name] || {}, controls: controlsObj }, { 'imports': { 'renderWidget' : renderWidget}});
        oneOffFile = oneOffFile.replace(/^\s*\n/gm, '');

        oneOffMD5 = md5(oneOffFile);
        fs.writeFileSync(oneOff, oneOffFile);
      } else {

        if(!force && fs.existsSync(directory)) {
          if(done) done(null, null, null);
          logger.error('Scaffolding for ' + name + ' already exists, use --force to overwrite');
          return false;
        }

        mkdirp.sync(directory);

        var template = _.template(individualTemplate, { widgetFiles: widgetFiles, typeName: name, typeInfo: typeInfo[name] || {}, controls: controlsObj }, { 'imports': { 'renderWidget' : renderWidget}});
        template = template.replace(/^\s*\n/gm, '');

        individualMD5 = md5(template);
        fs.writeFileSync(individual, template);

        var lTemplate = _.template(listTemplate, { typeName: name });

        listMD5 = md5(lTemplate);
        fs.writeFileSync(list, lTemplate);
      }

      if(done) done(individualMD5, listMD5, oneOffMD5);
    });

    return true;
  };

  /**
   * Send signal to local livereload server to reload files
   * @param  {Array}      files     List of files to reload
   * @param  {Function}   done      Callback passed either a true value to indicate its done, or an error
   */
  this.reloadFiles = function(done) {
    request({ url : 'http://localhost:' + liveReloadPort + '/changed?files=true', timeout: 10  }, function(error, response, body) {
      if(done) done(true);
    });
  };

  /**
   * Starts a live reload server, which will refresh the pages when signaled
   */
  this.startLiveReload = function() {
    tinylr().listen(liveReloadPort);
  };

  /**
   * Sends a message to the CMS through a websocket initiated by the CMS
   * @param  {String}      message    Message to send
   */
  this.sendSockMessage = function(message) {
    if(websocket) {
      websocket.send('message:' + JSON.stringify(message));
    }
  };

  var runCommand = function(cmd, cwd, args, pipe, cb) {
    if(typeof pipe == 'function') {
      cb = pipe;
      pipe = false;
    }

    var command = spawn(cmd, args, {
      stdio: [process.stdin, pipe ? 'pipe' : process.stdout, process.stderr],
      cwd: cwd
    });

    var output = '';

    if(pipe) {
      command.stdout.on('data', function(data) {
        output += data;
      });
    }

    command.on('close', function() {
      cb(output);
    })
  }

  var runNpm = function(cb) {
    if(options.npmCache) {
      runCommand(options.npm || 'npm', '.', ['config', 'get', 'cache'], true, function(diroutput) {
        var oldCacheDir = diroutput.trim();
        runCommand(options.npm || 'npm', '.', ['config', 'set', 'cache', options.npmCache], function() {
          runCommand(options.npm || 'npm', '.', ['install'], function() {
            runCommand(options.npm || 'npm', '.', ['config', 'set', 'cache', oldCacheDir], function() {
              cb();
            });
          });
        });
      });
    } else {
      runCommand(options.npm || 'npm', '.', ['install'], function() {
        console.log('NPM done');
        cb();
      });
    }
  };

  /**
   * Starts a websocket listener on 0.0.0.0 (for people who want to run wh serv over a network)
   * Accepts messages for generating scaffolding and downloading preset themes.
   */
  this.webListener = function() {
    var server = new ws({ host: '0.0.0.0', port: 6557 });

    server.on('connection', function(sock) {
      websocket = sock;

      var buildQueue = async.queue(function (task, callback) {
          self.buildBoth(function() {
            sock.send('done');
            callback();
          }, self.reloadFiles);
      }, 1);

      sock.on('message', function(message) {
        if(message.indexOf('scaffolding:') === 0)
        {
          var name = message.replace('scaffolding:', '');
          self.makeScaffolding(name, function(individualMD5, listMD5, oneOffMD5) {
            sock.send('done:' + JSON.stringify({ individualMD5: individualMD5, listMD5: listMD5, oneOffMD5: oneOffMD5 }));
          });
        } else if (message.indexOf('scaffolding_force:') === 0) {
          var name = message.replace('scaffolding_force:', '');
          self.makeScaffolding(name, function(individualMD5, listMD5, oneOffMD5) {
            sock.send('done:' + JSON.stringify({ individualMD5: individualMD5, listMD5: listMD5, oneOffMD5: oneOffMD5 }));
          }, true);
        } else if (message.indexOf('check_scaffolding:') === 0) {
          var name = message.replace('check_scaffolding:', '');
          self.checkScaffoldingMD5(name, function(individualMD5, listMD5, oneOffMD5) {
            sock.send('done:' + JSON.stringify({ individualMD5: individualMD5, listMD5: listMD5, oneOffMD5: oneOffMD5 }));
          });
        } else if (message === 'reset_files') {
          resetGenerator(function(error) {
            if(error) {
              sock.send('done:' + JSON.stringify({ err: 'Error while resetting files' }));
            } else {
              sock.send('done');
            }
          });
        } else if (message === 'supported_messages') {
          sock.send('done:' + JSON.stringify([
            'scaffolding', 'scaffolding_force', 'check_scaffolding', 'reset_files', 'supported_messages',
            'push', 'build', 'preset', 'layouts', 'preset_localv2', 'generate_slug_v2'
          ]));
        } else if (message.indexOf('generate_slug_v2:') === 0) {
          var obj = JSON.parse(message.replace('generate_slug_v2:', ''));
          var type = obj.type;
          var name = obj.name;
          var date = obj.date;

          getTypeData(type, function(typeInfo) {
            var tmpSlug = '';
            tmpSlug = slug(name).toLowerCase();

            if(typeInfo && typeInfo.customUrls && typeInfo.customUrls.individualUrl) {
              tmpSlug = utils.parseCustomUrl(typeInfo.customUrls.individualUrl, date) + '/' + tmpSlug;
            }

            if(typeInfo && typeInfo.customUrls && typeInfo.customUrls.listUrl) {

              if(typeInfo.customUrls.listUrl === '#') {
                tmpSlug = tmpSlug;
              } else {
                tmpSlug = typeInfo.customUrls.listUrl + '/' + tmpSlug;
              }
            } else {
              tmpSlug = type + '/' + tmpSlug;
            }

            sock.send('done:' + JSON.stringify(tmpSlug));
          });
        } else if (message === 'build') {
          buildQueue.push({}, function(err) {});
        } else if (message.indexOf('preset_local:') === 0) {
          var fileData = message.replace('preset_local:', '');

          if(!fileData) {
            sock.send('done');
            return;
          }

          extractPresetLocal(fileData, function(data) {
            runNpm(function() {
              sock.send('done:' + JSON.stringify(data));
            });
          });
        } else if (message.indexOf('preset:') === 0) {
          var url = message.replace('preset:', '');
          if(!url) {
            sock.send('done');
            return;
          }
          downloadPreset(url, function(data) {
            runNpm(function() {
              console.log('DONE RUNNING?')
              sock.send('done:' + JSON.stringify(data));
            });
          });
        } else {
          sock.send('done');
        }
      });
    });
  };

  /**
   * Inintializes firebase configuration for a new site
   * @param  {String}    sitename  Name of site to generate config for
   * @param  {String}    secretkey Secret key for the site (gotten from firebase)
   * @param  {Boolean}   copyCms   True if the CMS should be overwritten, false otherwise
   * @param  {Function}  done      Callback to call when operation is done
   */
  this.init = function(sitename, secretkey, copyCms, firebase, done) {
    var oldConf = config.get('webhook');

    var confFile = fs.readFileSync('./libs/.firebase.conf.jst');

    if(firebase) {
      confFile = fs.readFileSync('./libs/.firebase-custom.conf.jst');
    }

    var noSearch = null;

    if(oldConf.noSearch !== null && typeof oldConf.noSearch !== 'undefined') {
      noSearch = oldConf.noSearch;
    }

    // TODO: Grab bucket information from server eventually, for now just use the site name
    var templated = _.template(confFile, { secretKey: secretkey, siteName: sitename, firebase: firebase, embedlyKey: oldConf.embedly || 'your-embedly-key', serverAddr: oldConf.server || 'your-server-address', noSearch: noSearch });

    fs.writeFileSync('./.firebase.conf', templated);

    if(copyCms) {
      var cmsFile = fs.readFileSync('./libs/cms.html');

      var cmsTemplated = _.template(cmsFile, { siteName: sitename });

      mkdirp.sync('./pages/');

      fs.writeFileSync('./pages/cms.html', cmsTemplated);
    }

    done(true);
  };

  /**
   * Sets up asset generation (automatic versioning) for pushing to production
   * @param  {Object}    grunt  Grunt object from generatorTasks
   */
  this.assets = function(grunt, done) {

    removeDirectory('.whdist', function() {

      mkdirp.sync('.whdist');

      var files = wrench.readdirSyncRecursive('pages');

      files.forEach(function(file) {
        var originalFile = 'pages/' + file;
        var destFile = '.whdist/pages/' + file;

        if(!fs.lstatSync(originalFile).isDirectory())
        {
          var content = fs.readFileSync(originalFile);

          if(path.extname(originalFile) === '.html') {
            content = content.toString();
            content = content.replace('\r\n', '\n').replace('\r', '\n');
          }

          mkdirp.sync(path.dirname(destFile));
          fs.writeFileSync(destFile, content);
        }
      });

      files = wrench.readdirSyncRecursive('templates');

      files.forEach(function(file) {
        var originalFile = 'templates/' + file;
        var destFile = '.whdist/templates/' + file;

        if(!fs.lstatSync(originalFile).isDirectory())
        {
          var content = fs.readFileSync(originalFile);

          if(path.extname(originalFile) === '.html') {
            content = content.toString();
            content = content.replace('\r\n', '\n').replace('\r', '\n');
          }

          mkdirp.sync(path.dirname(destFile));
          fs.writeFileSync(destFile, content);
        }
      });

      files = wrench.readdirSyncRecursive('static');

      files.forEach(function(file) {
        var originalFile = 'static/' + file;
        var destFile = '.whdist/static/' + file;

        if(!fs.lstatSync(originalFile).isDirectory())
        {
          var content = fs.readFileSync(originalFile);

          if(path.extname(originalFile) === '.html') {
            content = content.toString();
            content = content.replace('\r\n', '\n').replace('\r', '\n');
          }

          mkdirp.sync(path.dirname(destFile));
          fs.writeFileSync(destFile, content);
        }
      });

      grunt.task.run('useminPrepare');
      grunt.task.run('assetsMiddle');

      done();
    });

  }

  /**
   * Run asset versioning software if configs exist for them
   * @param  {Object}    grunt  Grunt object from generatorTasks
   */
  this.assetsMiddle = function(grunt) {

    grunt.option('force', true);

    if(!_.isEmpty(grunt.config.get('concat')))
    {
      grunt.task.run('concat');
    }

    if(!_.isEmpty(grunt.config.get('uglify')))
    {
      grunt.task.run('uglify');
    }

    if(!_.isEmpty(grunt.config.get('cssmin')))
    {
      grunt.task.run('cssmin');
    }

    grunt.task.run('rev');
    grunt.task.run('usemin');
    grunt.task.run('assetsAfter');

  }

  /**
   * Finish asset versioning
   * @param  {Object}    grunt  Grunt object from generatorTasks
   */
  this.assetsAfter = function(grunt, done) {
    removeDirectory('.tmp', function() {
      var files = wrench.readdirSyncRecursive('static');

      files.forEach(function(file) {
        var filePath = 'static/' + file;
        var distPath = '.whdist/static/' + file;
        if(!fs.lstatSync(filePath).isDirectory() && !fs.existsSync(distPath)) {
          var fileData = fs.readFileSync(filePath);
          fs.writeFileSync(distPath, fileData);
        }
      });

      done();
    });
  }

  /**
   * Enables strict mode, exceptions cause full crash, normally for production (so bad generators do not ruin sites)
   */
  this.enableStrictMode = function() {
    strictMode = true;
  }

  this.enableProduction = function() {
    productionFlag = true;
  }

  return this;
};


text:
using System;

namespace Mozu.Api.Contracts.Core
{
		///
		///	Information required to authenticate a user.
		///
		public class UserAuthInfo
		{
			public string EmailAddress { get; set; }

			public string Password { get; set; }

		}

}


text:
package org.apache.hadoop.hdfs.notifier.server;

public interface IServerDispatcher extends Runnable {

  public void assignClient(long clientId);

  public void removeClient(long clientId);
}


text:
@interface SchoolBusDetailViewController : ZJUBaseTableViewController
@property (nonatomic, strong) ZJUSchoolBusDataItem *item;
@end


text:
#region Apache License Version 2.0

#endregion Apache License Version 2.0

using System;
using System.Collections.Generic;
using System.Linq;
/*----------------------------------------------------------------
    Copyright (C) 2017 Senparc

    文件名:ResponseMessageNoResponse.cs
    文件功能描述:无需响应(回复空字符串)的响应类型

    创建标识:Senparc - 20150505

----------------------------------------------------------------*/
using System.Text;
using System.Threading.Tasks;
using Senparc.Weixin.Entities;

namespace Senparc.Weixin.MP.Entities
{
    /// <summary>
    /// 当MessageHandler接收到IResponseNothing的返回类型参数时,只会向微信服务器返回空字符串,等同于return null
    /// </summary>
    public class ResponseMessageNoResponse : ResponseMessageBase,IResponseMessageNoResponse
    {
    }
}


text:
from django.test import RequestFactory
from django.test.utils import override_settings
from django.urls import reverse

from monitoring.overview import constants
from monitoring.overview import views
from monitoring.test import helpers

INDEX_URL = reverse(
    constants.URL_PREFIX + 'index')

class OverviewTest(helpers.TestCase):

    @override_settings(DASHBOARDS=[],
                       KIBANA_POLICY_SCOPE='monitoring',
                       KIBANA_POLICY_RULE='monitoring:kibana_access',
                       ENABLE_LOG_MANAGEMENT_BUTTON=False,
                       ENABLE_EVENT_MANAGEMENT_BUTTON=False,
                       SHOW_GRAFANA_HOME=True)
    def test_index_get(self):
        res = self.client.get(INDEX_URL)
        self.assertTemplateUsed(
            res, 'monitoring/overview/index.html')
        self.assertTemplateUsed(res, 'monitoring/overview/monitor.html')

class KibanaProxyViewTest(helpers.TestCase):

    def setUp(self):
        super(KibanaProxyViewTest, self).setUp()
        self.view = views.KibanaProxyView()
        self.request_factory = RequestFactory()

    def test_get_relative_url_with_unicode(self):
        """Tests if it properly converts multibyte characters."""
        from urllib import parse as urlparse

        self.view.request = self.request_factory.get(
            '/', data={'a': 1, 'b': 2}
        )
        expected_path = ('/elasticsearch/.kibana/search'
                         '/New-Saved-Search%E3%81%82')
        expected_qs = {'a': ['1'], 'b': ['2']}

        url = self.view.get_relative_url(
            u'/elasticsearch/.kibana/search/New-Saved-Searchあ'
        )
        # order of query params may change
        parsed_url = urlparse.urlparse(url)
        actual_path = parsed_url.path
        actual_qs = urlparse.parse_qs(parsed_url.query)

        self.assertEqual(actual_path, expected_path)
        self.assertEqual(actual_qs, expected_qs)


text: <!DOCTYPE html>