text:
import { CommandLineInputs, CommandLineOptions, MetadataGroup, combine, contains, validators } from '@ionic/cli-framework';
import { LOGGER_LEVELS } from '@ionic/cli-framework-output';
import { tmpfilepath } from '@ionic/utils-fs';
import { columnar } from '@ionic/utils-terminal';
import { sleep } from '@ionic/utils-process';
import * as chalk from 'chalk';
import * as Debug from 'debug';
import * as fs from 'fs';

import { CommandMetadata } from '../../definitions';
import { isSuperAgentError } from '../../guards';
import { input, strong, weak } from '../../lib/color';
import { Command } from '../../lib/command';
import { FatalException } from '../../lib/errors';
import { fileUtils } from '../../lib/utils/file';
import { createRequest, download } from '../../lib/utils/http';
import { IONIC_CLOUD_CLI_MIGRATION } from '../../lib/updates';

const debug = Debug('ionic:commands:package:build');
const PLATFORMS = ['android', 'ios'];

const ANDROID_BUILD_TYPES = ['debug', 'release'];
const IOS_BUILD_TYPES = ['development', 'ad-hoc', 'app-store', 'enterprise'];
const APP_STORE_COMPATIBLE_TYPES = ['release', 'app-store', 'enterprise'];
const BUILD_TYPES = ANDROID_BUILD_TYPES.concat(IOS_BUILD_TYPES);
const ANDROID_ARTIFACT_TYPES = ['aab', 'apk'];
const IOS_ARTIFACT_TYPES = ['ipa', 'dsym'];
const ARTIFACT_TYPES = ANDROID_ARTIFACT_TYPES.concat(IOS_ARTIFACT_TYPES);
const TARGET_PLATFORM = ['Android', 'iOS - Xcode 11 (Preferred)', 'iOS - Xcode 10'];

interface Artifact {
  name: string;
  url: string;
  artifact_type: string;
}

export interface PackageBuild {
  job_id: number;
  id: string;
  caller_id: number;
  platform: string;
  build_type: string;
  created: string;
  finished: string;
  state: string;
  commit: any;
  stack: any;
  profile_tag: string;
  automation_id: number;
  environment_id: number;
  native_config_id: number;
  automation_name: string;
  environment_name: string;
  native_config_name: string;
  distribution_credential_name: string;
  job: any;
  distribution_trace: string;
  artifacts: Artifact[];
}

interface DownloadUrl {
  url: string | null;
}

export class BuildCommand extends Command {
  async getMetadata(): Promise<CommandMetadata> {
    const dashUrl = this.env.config.getDashUrl();

    return {
      name: 'build',
      type: 'project',
      groups: [MetadataGroup.PAID, MetadataGroup.DEPRECATED],
      summary: 'Create a package build on Appflow',
      description: `
This command creates a package build on Appflow. While the build is running, it prints the remote build log to the terminal. If the build is successful, it downloads the created app package file in the current directory. Downloading build artifacts can be skipped by supplying the flag ${input('skip-download')}.

Apart from ${input('--commit')}, every option can be specified using the full name setup within the Dashboard[^dashboard].

The ${input('--signing-certificate')} option is mandatory for any iOS build but not for Android debug builds.

Customizing the build:
- The ${input('--environment')} and ${input('--native-config')} options can be used to customize the groups of values exposed to the build.
- Override the preferred platform with ${input('--build-stack')}. This is useful for building older iOS apps.

Deploying the build to an App Store:
- The ${input('--destination')} option can be used to deliver the app created by the build to the configured App Store. \
This can be used only together with build type ${input('release')} for Android and build types ${input('app-store')} or ${input('enterprise')} for iOS.

Downloading build artifacts:
- By default once the build is complete, all artifacts are downloaded for the selected platform. ${input('aab')} and ${input('apk')} for Android \
${input('ipa')} and ${input('dsym')} for iOS.
- The ${input('--artifact-type')} option can be used to limit artifact downloads to only of that type. For instance, with Android, you can specify ${input('aab')} \
if you do not wish to download ${input('apk')}.
`,
      footnotes: [
        {
          id: 'dashboard',
          url: dashUrl,
        },
      ],
      exampleCommands: [
        'android debug',
        'ios development --signing-certificate="iOS Signing Certificate Name"',
        'android debug --environment="My Custom Environment Name"',
        'android debug --native-config="My Custom Native Config Name"',
        'android debug --commit=2345cd3305a1cf94de34e93b73a932f25baac77c',
        'android debug --artifact-type=aab',
        'android debug --skip-download',
        'android debug --aab-name="my-app-prod.aab" --apk-name="my-app-prod.apk"',
        'ios development --signing-certificate="iOS Signing Certificate Name" --build-stack="iOS - Xcode 9"',
        'ios development --signing-certificate="iOS Signing Certificate Name" --ipa-name=my_custom_file_name.ipa',
        'ios app-store --signing-certificate="iOS Signing Certificate Name" --destination="Apple App Store Destination"',
      ],
      inputs: [
        {
          name: 'platform',
          summary: `The platform to package (${PLATFORMS.map(v => input(v)).join(', ')})`,
          validators: [validators.required, contains(PLATFORMS, {})],
        },
        {
          name: 'type',
          summary: `The build type (${BUILD_TYPES.map(v => input(v)).join(', ')})`,
          validators: [validators.required, contains(BUILD_TYPES, {})],
        },
      ],
      options: [
        {
          name: 'signing-certificate',
          summary: 'Signing certificate',
          type: String,
          spec: { value: 'name' },
        },
        {
          name: 'environment',
          summary: 'The group of environment variables exposed to your build',
          type: String,
          spec: { value: 'name' },
        },
        {
          name: 'native-config',
          summary: 'The group of native config variables exposed to your build',
          type: String,
          spec: { value: 'name' },
        },
        {
          name: 'destination',
          summary: 'The configuration to deploy the build artifact to the app store',
          type: String,
          spec: { value: 'name' },
        },
        {
          name: 'commit',
          summary: 'Commit (defaults to HEAD)',
          type: String,
          groups: [MetadataGroup.ADVANCED],
          spec: { value: 'sha1' },
        },
        {
          name: 'build-stack',
          summary: `Target platform (${TARGET_PLATFORM.map(v => input(`"${v}"`)).join(', ')})`,
          type: String,
          groups: [MetadataGroup.ADVANCED],
          spec: { value: 'name' },
        },
        {
          name: 'build-file-name',
          summary: 'The name for the downloaded build file',
          type: String,
          groups: [MetadataGroup.DEPRECATED],
          spec: { value: 'name' },
        },
        {
          name: 'ipa-name',
          summary: 'The name for the downloaded ipa file',
          type: String,
          groups: [MetadataGroup.ADVANCED],
          spec: { value: 'name' },
        },
        {
          name: 'dsym-name',
          summary: 'The name for the downloaded dsym file',
          type: String,
          groups: [MetadataGroup.ADVANCED],
          spec: { value: 'name' },
        },
        {
          name: 'apk-name',
          summary: 'The name for the downloaded apk file',
          type: String,
          groups: [MetadataGroup.ADVANCED],
          spec: { value: 'name' },
        },
        {
          name: 'aab-name',
          summary: 'The name for the downloaded aab file',
          type: String,
          groups: [MetadataGroup.ADVANCED],
          spec: { value: 'name' },
        },
        {
          name: 'artifact-type',
          summary: `The artifact type (${ARTIFACT_TYPES.map(v => input(v)).join(', ')})`,
          type: String,
          spec: { value: 'name' },
        },
        {
          name: 'skip-download',
          summary: `Skip downloading build artifacts after command succeeds.`,
          type: Boolean,
          spec: { value: 'name' },
          default: false,
        },
      ],
    };
  }

  async preRun(inputs: CommandLineInputs, options: CommandLineOptions): Promise<void> {
    this.env.log.warn(IONIC_CLOUD_CLI_MIGRATION);

    if (!inputs[0]) {
      const platformInput = await this.env.prompt({
        type: 'list',
        name: 'platform',
        choices: PLATFORMS,
        message: `Platform to package:`,
        validate: v => combine(validators.required, contains(PLATFORMS, {}))(v),
      });

      inputs[0] = platformInput;
    }

    const buildTypes = inputs[0] === 'ios' ? IOS_BUILD_TYPES : ANDROID_BUILD_TYPES;

    // validate that the build type is valid for the platform
    let reenterBuildType = false;
    if (inputs[1] && !buildTypes.includes(inputs[1])) {
      reenterBuildType = true;
      this.env.log.nl();
      this.env.log.warn(`Build type ${strong(inputs[1])} incompatible for ${strong(inputs[0])}; please choose a correct one`);
    }

    if (!inputs[1] || reenterBuildType) {
      const typeInput = await this.env.prompt({
        type: 'list',
        name: 'type',
        choices: buildTypes,
        message: `Build type:`,
        validate: v => combine(validators.required, contains(buildTypes, {}))(v),
      });

      inputs[1] = typeInput;
    }

    if (options['target-platform']) {
      this.env.log.warn(`The ${input('--target-platform')} option has been deprecated. Please use ${input('--build-stack')}.`);
      options['build-stack'] = options['target-platform'];
    }

    if (options['security-profile']) {
      this.env.log.warn(`The ${input('--security-profile')} option has been deprecated. Please use ${input('--signing-certificate')}.`);
      options['signing-certificate'] = options['security-profile'];
    }

    // the signing certificate is mandatory for iOS packages, so prompting if it is missing
    if (inputs[0] === 'ios' && !options['signing-certificate']) {
      if (this.env.flags.interactive) {
        this.env.log.nl();
        this.env.log.warn(`A signing certificate is mandatory to build an iOS package`);
      }

      const securityProfileOption = await this.env.prompt({
        type: 'input',
        name: 'signing-certificate',
        message: `Signing Certificate Name:`,
      });

      options['signing-certificate'] = securityProfileOption;
    }

    // if destination is present, validate that a proper build type has been been specified
    if (options['destination'] && !APP_STORE_COMPATIBLE_TYPES.includes(inputs[1])) {
      throw new FatalException(`Build with type ${strong(String(inputs[1]))} cannot be deployed to App Store`);
    }

    if (options['artifact-type'] && (Array.isArray(options['artifact-type']) || typeof options['artifact-type'] === 'string')) {
      const artifactTypes = Array.isArray(options['artifact-type']) ? options['artifact-type'] : [options['artifact-type']];
      let unsupported: string[];

      switch (inputs[0]) {
        case 'android':
          unsupported = artifactTypes.filter(type => !ANDROID_ARTIFACT_TYPES.includes(type));
          break;
        case 'ios':
          unsupported = artifactTypes.filter(type => !IOS_ARTIFACT_TYPES.includes(type));
          break;
        default:
          throw new FatalException(`Unsupported platform ${inputs[0]}`);
      }

      if (unsupported.length) {
        throw new FatalException(`Unsupported artifact types for platform ${inputs[0]}: ${[...unsupported]}`);
      }
    }
  }

  async run(inputs: CommandLineInputs, options: CommandLineOptions): Promise<void> {
    if (!this.project) {
      throw new FatalException(`Cannot run ${input('ionic package build')} outside a project directory.`);
    }

    const token = await this.env.session.getUserToken();
    const appflowId = await this.project.requireAppflowId();
    const [ platform, buildType ] = inputs;

    if (!options.commit) {
      options.commit = (await this.env.shell.output('git', ['rev-parse', 'HEAD'], { cwd: this.project.directory })).trim();
      debug(`Commit hash: ${strong(options.commit)}`);
    }

    let build = await this.createPackageBuild(appflowId, token, platform, buildType, options);
    const buildId = build.job_id;

    const details = columnar([
      ['App ID', strong(appflowId)],
      ['Build ID', strong(buildId.toString())],
      ['Commit', strong(`${build.commit.sha.substring(0, 6)} ${build.commit.note}`)],
      ['Target Platform', strong(build.stack.friendly_name)],
      ['Build Type', strong(build.build_type)],
      ['Artifact Type(s)', options['artifact-type'] ? strong(`${options['artifact-type']}`.toUpperCase()) : weak('all available')],
      ['Security Profile', build.profile_tag ? strong(build.profile_tag) : weak('not set')],
      ['Environment', build.environment_name ? strong(build.environment_name) : weak('not set')],
      ['Native Config', build.native_config_name ? strong(build.native_config_name) : weak('not set')],
      ['Destination', build.distribution_credential_name ? strong(build.distribution_credential_name) : weak('not set')],
    ], { vsep: ':' });

    this.env.log.ok(
      `Build created\n` +
      details + '\n\n'
    );

    build = await this.tailBuildLog(appflowId, buildId, token);
    if (build.state !== 'success') {
      throw new Error(`Build ${build.state}`);
    }

    if (options['skip-download']) {
      return;
    }

    const availableArtifactTypes = build.artifacts.map((artifact) => artifact.artifact_type);

    if (Array.isArray(options['artifact-type'])) {
      let errors = [];
      for (const artifactType of new Set(options['artifact-type'])) {
        try {
          await this.downloadArtifact(appflowId, buildId, artifactType, token, options);
        } catch (error) {
          errors.push(error);
        }
      }

      if (errors.length) {
        throw new FatalException(`There were issues downloading artifacts: ${errors.join('\n')}`);
      }

    } else if (typeof options['artifact-type'] == 'string') {
      await this.downloadArtifact(appflowId, buildId, options['artifact-type'], token, options);
    } else {
      for (const artifactType of availableArtifactTypes) {
        await this.downloadArtifact(appflowId, buildId, artifactType.toUpperCase(), token, options);
      }
    }
  }

  async sanitizeString(value: string | string[] | boolean | null | undefined): Promise<string> {

    if (!value || typeof (value) !== 'string') {
      return '';
    }

    if (!fileUtils.isValidFileName(value)) {
      throw new FatalException(`${strong(String(value))} is not a valid file name`);
    }

    return String(value);
  }

  async createPackageBuild(appflowId: string, token: string, platform: string, buildType: string, options: CommandLineOptions): Promise<PackageBuild> {
    const { req } = await this.env.client.make('POST', `/apps/${appflowId}/packages/verbose_post`);
    req.set('Authorization', `Bearer ${token}`).send({
      platform,
      build_type: buildType,
      commit_sha: options.commit,
      stack_name: options['build-stack'],
      profile_name: options['signing-certificate'],
      environment_name: options.environment,
      native_config_name: options['native-config'],
      distribution_credential_name: options.destination,
    });

    try {
      const res = await this.env.client.do(req);
      return res.data as PackageBuild;
    } catch (e) {
      if (isSuperAgentError(e)) {
        if (e.response.status === 401) {
          this.env.log.error('Try logging out and back in again.');
        }
        const apiErrorMessage = (e.response.body.error && e.response.body.error.message) ? e.response.body.error.message : 'Api Error';
        throw new FatalException(`Unable to create build: ` + apiErrorMessage);
      } else {
        throw e;
      }
    }
  }

  async getPackageBuild(appflowId: string, buildId: number, token: string): Promise<PackageBuild> {
    const { req } = await this.env.client.make('GET', `/apps/${appflowId}/packages/${buildId}`);
    req.set('Authorization', `Bearer ${token}`).send();

    try {
      const res = await this.env.client.do(req);
      return res.data as PackageBuild;
    } catch (e) {
      if (isSuperAgentError(e)) {
        if (e.response.status === 401) {
          this.env.log.error('Try logging out and back in again.');
        }
        const apiErrorMessage = (e.response.body.error && e.response.body.error.message) ? e.response.body.error.message : 'Api Error';
        throw new FatalException(`Unable to get build ${buildId}: ` + apiErrorMessage);
      } else {
        throw e;
      }
    }
  }

  async getDownloadUrl(appflowId: string, buildId: number, artifactType: string, token: string): Promise<DownloadUrl> {
    const { req } = await this.env.client.make('GET', `/apps/${appflowId}/packages/${buildId}/download?artifact_type=${artifactType}`);
    req.set('Authorization', `Bearer ${token}`).send();

    try {
      const res = await this.env.client.do(req);
      return res.data as DownloadUrl;
    } catch (e) {
      if (isSuperAgentError(e)) {
        if (e.response.status === 401) {
          this.env.log.error('Try logging out and back in again.');
        }
        const apiErrorMessage = (e.response.body.error && e.response.body.error.message) ? e.response.body.error.message : 'Api Error';
        throw new FatalException(`Unable to get download URL for build ${buildId}: ` + apiErrorMessage);
      } else {
        throw e;
      }
    }
  }

  async tailBuildLog(appflowId: string, buildId: number, token: string): Promise<PackageBuild> {
    let build;
    let start = 0;
    const ws = this.env.log.createWriteStream(LOGGER_LEVELS.INFO, false);

    let isCreatedMessage = false;
    let errorsEncountered = 0;
    while (!(build && ['success', 'failed', 'canceled'].includes(build.state))) {
      try {
        await sleep(5000);
        build = await this.getPackageBuild(appflowId, buildId, token);
        if (build && build.state === 'created' && !isCreatedMessage) {
          ws.write(chalk.yellow('Concurrency limit reached: build will start as soon as other builds finish.'));
          isCreatedMessage = true;
        }
        const trace = build.job.trace;
        if (trace.length > start) {
          ws.write(trace.substring(start));
          start = trace.length;
        }
        errorsEncountered = 0;
      } catch (e) {
        // Retry up to 3 times in the case of an error.
        errorsEncountered++;
        ws.write(chalk.yellow(`Encountered error: ${e} while fetching build data retrying.`));
        if (errorsEncountered >= 3) {
          ws.write(chalk.red(`Encountered ${errorsEncountered} errors in a row. Job will now fail.`));
          throw e;
        }
      }
    }
    ws.end();

    return build;
  }

  async downloadBuild(url: string, filename: string): Promise<string> {
    const { req } = await createRequest('GET', url, this.env.config.getHTTPConfig());

    if (!filename) {
      req.on('response', res => {
        const contentDisposition = res.header['content-disposition'];
        filename = contentDisposition ? contentDisposition.split('=')[1].replace(/([/?<>*|\"])/g, '_') : 'output.bin';
      });
    }

    const tmpFile = tmpfilepath('ionic-package-build');
    const ws = fs.createWriteStream(tmpFile);
    await download(req, ws, {});
    fs.copyFileSync(tmpFile, filename);
    fs.unlinkSync(tmpFile);
    return filename;
  }

  async downloadArtifact(appflowId: string, buildId: number, artifactType: string, token: string, options: CommandLineOptions) {

    const url = await this.getDownloadUrl(appflowId, buildId, artifactType.toUpperCase(), token);

    if (!url.url) {
      throw new Error(`Artifact type '${artifactType}' not found`);
    }

    let customBuildFileName = '';

    if (options['build-file-name']) {
      this.env.log.warn(`The ${input('--build-file-name')} option has been deprecated. Please use ${input('--ipa-name')}, ${input('--apk-name')} or ${input('--{ artifact }-name')}.`);
      customBuildFileName = await this.sanitizeString(options['build-file-name']);
    } else {
      customBuildFileName = await this.sanitizeString(options[`${artifactType.toLowerCase()}-name`]);
    }

    const filename = await this.downloadBuild(url.url, customBuildFileName);
    this.env.log.ok(`Artifact downloaded: ${filename}`);
  }
}


text:
set -e

export BQ_PROJECT=$1  # project
export BQ_DATASET=$2  # dataset

TABLE_LIST=$(bq ls -n 1000 "$BQ_PROJECT:$BQ_DATASET")
if [[ "$TABLE_LIST" == *'visit_detail'* ]];
then
  echo "Dataset is OMOP 5.3.1, dataset contains 'visit_detail' table"
fi
################################################
# INSERT DATA
################################################
echo "ds_condition_occurrence - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_condition_occurrence\`
    (PERSON_ID, CONDITION_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY,
    CONDITION_START_DATETIME, CONDITION_END_DATETIME, CONDITION_TYPE_CONCEPT_ID, CONDITION_TYPE_CONCEPT_NAME,
    STOP_REASON, VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, CONDITION_SOURCE_VALUE, CONDITION_SOURCE_CONCEPT_ID,
    SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, CONDITION_STATUS_SOURCE_VALUE,
    CONDITION_STATUS_CONCEPT_ID, CONDITION_STATUS_CONCEPT_NAME)
select a.PERSON_ID, CONDITION_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
    c1.vocabulary_id as STANDARD_VOCABULARY, CONDITION_START_DATETIME, CONDITION_END_DATETIME, CONDITION_TYPE_CONCEPT_ID,
    c2.concept_name as CONDITION_TYPE_CONCEPT_NAME, STOP_REASON, a.VISIT_OCCURRENCE_ID, c3.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME,
    CONDITION_SOURCE_VALUE, CONDITION_SOURCE_CONCEPT_ID, c4.concept_name as SOURCE_CONCEPT_NAME, c4.concept_code as SOURCE_CONCEPT_CODE,
    c4.vocabulary_id as SOURCE_VOCABULARY, CONDITION_STATUS_SOURCE_VALUE, CONDITION_STATUS_CONCEPT_ID,
    c5.concept_name as CONDITION_STATUS_CONCEPT_NAME
from \`$BQ_PROJECT.$BQ_DATASET.condition_occurrence\` a
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.CONDITION_CONCEPT_ID = c1.CONCEPT_ID
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.CONDITION_TYPE_CONCEPT_ID = c2.CONCEPT_ID
left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.VISIT_OCCURRENCE_ID = v.VISIT_OCCURRENCE_ID
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on v.visit_concept_id = c3.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on a.CONDITION_SOURCE_CONCEPT_ID = c4.CONCEPT_ID
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.CONDITION_STATUS_CONCEPT_ID = c5.CONCEPT_ID"

echo "ds_device - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_device\`
     (PERSON_ID, DEVICE_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY,
     DEVICE_EXPOSURE_START_DATETIME, DEVICE_EXPOSURE_END_DATETIME, DEVICE_TYPE_CONCEPT_ID, DEVICE_TYPE_CONCEPT_NAME,
     VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, DEVICE_SOURCE_VALUE, DEVICE_SOURCE_CONCEPT_ID,
     SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY)
 select a.PERSON_ID, DEVICE_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
     c1.vocabulary_id as STANDARD_VOCABULARY, DEVICE_EXPOSURE_START_DATETIME, DEVICE_EXPOSURE_END_DATETIME, DEVICE_TYPE_CONCEPT_ID,
     c2.concept_name as DEVICE_TYPE_CONCEPT_NAME, a.VISIT_OCCURRENCE_ID, c3.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME,
     DEVICE_SOURCE_VALUE, DEVICE_SOURCE_CONCEPT_ID, c4.concept_name as SOURCE_CONCEPT_NAME, c4.concept_code as SOURCE_CONCEPT_CODE,
     c4.vocabulary_id as SOURCE_VOCABULARY
 from \`$BQ_PROJECT.$BQ_DATASET.device_exposure\` a
 left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.DEVICE_CONCEPT_ID = c1.CONCEPT_ID
 left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.DEVICE_TYPE_CONCEPT_ID = c2.CONCEPT_ID
 left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.VISIT_OCCURRENCE_ID = v.VISIT_OCCURRENCE_ID
 left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on v.visit_concept_id = c3.concept_id
 left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on a.DEVICE_SOURCE_CONCEPT_ID = c4.CONCEPT_ID"

echo "ds_drug_exposure - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_drug_exposure\`
    (PERSON_ID, DRUG_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY, DRUG_EXPOSURE_START_DATETIME,
    DRUG_EXPOSURE_END_DATETIME, VERBATIM_END_DATE, DRUG_TYPE_CONCEPT_ID, DRUG_TYPE_CONCEPT_NAME, STOP_REASON, REFILLS, QUANTITY,
    DAYS_SUPPLY, SIG, ROUTE_CONCEPT_ID, ROUTE_CONCEPT_NAME, LOT_NUMBER, VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, DRUG_SOURCE_VALUE,
    DRUG_SOURCE_CONCEPT_ID, SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, ROUTE_SOURCE_VALUE, DOSE_UNIT_SOURCE_VALUE)
SELECT
    a.PERSON_ID, DRUG_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as CONCEPT_CODE,
    c1.vocabulary_id as STANDARD_VOCABULARY, DRUG_EXPOSURE_START_DATETIME, DRUG_EXPOSURE_END_DATETIME, VERBATIM_END_DATE,
    DRUG_TYPE_CONCEPT_ID, c2.concept_name as DRUG_TYPE_CONCEPT_NAME, STOP_REASON, REFILLS, QUANTITY, DAYS_SUPPLY, SIG,
    ROUTE_CONCEPT_ID, c3.concept_name as ROUTE_CONCEPT_NAME, LOT_NUMBER, a.VISIT_OCCURRENCE_ID,
    c4.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME, DRUG_SOURCE_VALUE, DRUG_SOURCE_CONCEPT_ID,
    c5.concept_name as SOURCE_CONCEPT_NAME, c5.concept_code as SOURCE_CONCEPT_CODE, c5.vocabulary_id as SOURCE_VOCABULARY,
    ROUTE_SOURCE_VALUE, DOSE_UNIT_SOURCE_VALUE
FROM \`$BQ_PROJECT.$BQ_DATASET.drug_exposure\` a
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.DRUG_CONCEPT_ID = c1.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.drug_type_concept_id = c2.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.ROUTE_CONCEPT_ID = c3.CONCEPT_ID
left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.VISIT_OCCURRENCE_ID = v.VISIT_OCCURRENCE_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on v.VISIT_CONCEPT_ID = c4.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.DRUG_SOURCE_CONCEPT_ID = c5.CONCEPT_ID"

echo "ds_measurement - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_measurement\`
    (PERSON_ID, MEASUREMENT_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY,
    MEASUREMENT_DATETIME, MEASUREMENT_TYPE_CONCEPT_ID, MEASUREMENT_TYPE_CONCEPT_NAME, OPERATOR_CONCEPT_ID,
    OPERATOR_CONCEPT_NAME, VALUE_AS_NUMBER, VALUE_AS_CONCEPT_ID, VALUE_AS_CONCEPT_NAME, UNIT_CONCEPT_ID, UNIT_CONCEPT_NAME,
    RANGE_LOW, RANGE_HIGH, VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, MEASUREMENT_SOURCE_VALUE,
    MEASUREMENT_SOURCE_CONCEPT_ID, SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, UNIT_SOURCE_VALUE,
    VALUE_SOURCE_VALUE)
select
    a.PERSON_ID, MEASUREMENT_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
    c1.vocabulary_id as STANDARD_VOCABULARY, MEASUREMENT_DATETIME, MEASUREMENT_TYPE_CONCEPT_ID,
    c2.concept_name as MEASUREMENT_TYPE_CONCEPT_NAME, OPERATOR_CONCEPT_ID, c3.concept_name as OPERATOR_CONCEPT_NAME,
    VALUE_AS_NUMBER, VALUE_AS_CONCEPT_ID, c4.concept_name as VALUE_AS_CONCEPT_NAME, UNIT_CONCEPT_ID,
    c5.concept_name as UNIT_CONCEPT_NAME, RANGE_LOW, RANGE_HIGH, a.VISIT_OCCURRENCE_ID,
    c6.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME, MEASUREMENT_SOURCE_VALUE, MEASUREMENT_SOURCE_CONCEPT_ID,
    c7.concept_name as SOURCE_CONCEPT_NAME, c7.concept_code as SOURCE_CONCEPT_CODE, c7.vocabulary_id as SOURCE_VOCABULARY,
    UNIT_SOURCE_VALUE, VALUE_SOURCE_VALUE
from \`$BQ_PROJECT.$BQ_DATASET.measurement\` a
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.measurement_concept_id = c1.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.measurement_type_concept_id = c2.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.operator_concept_id = c3.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on a.value_as_concept_id = c4.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.unit_concept_id = c5.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.visit_occurrence_id = v.visit_occurrence_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c6 on v.visit_concept_id = c6.concept_id
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c7 on a.measurement_source_concept_id = c7.concept_id"

echo "ds_observation - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_observation\`
    (PERSON_ID, OBSERVATION_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY,
    OBSERVATION_DATETIME, OBSERVATION_TYPE_CONCEPT_ID, OBSERVATION_TYPE_CONCEPT_NAME, VALUE_AS_NUMBER, VALUE_AS_STRING,
    VALUE_AS_CONCEPT_ID, VALUE_AS_CONCEPT_NAME, QUALIFIER_CONCEPT_ID, QUALIFIER_CONCEPT_NAME, UNIT_CONCEPT_ID,
    UNIT_CONCEPT_NAME, VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, OBSERVATION_SOURCE_VALUE,
    OBSERVATION_SOURCE_CONCEPT_ID, SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, UNIT_SOURCE_VALUE,
    QUALIFIER_SOURCE_VALUE, value_source_concept_id, value_source_value, questionnaire_response_id)
SELECT
    a.PERSON_ID, OBSERVATION_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
    c1.vocabulary_id as STANDARD_VOCABULARY, OBSERVATION_DATETIME, OBSERVATION_TYPE_CONCEPT_ID,
    c2.concept_name as OBSERVATION_TYPE_CONCEPT_NAME, VALUE_AS_NUMBER, VALUE_AS_STRING, VALUE_AS_CONCEPT_ID,
    c3.concept_name as VALUE_AS_CONCEPT_NAME, QUALIFIER_CONCEPT_ID, c4.concept_name as QUALIFIER_CONCEPT_NAME,
    UNIT_CONCEPT_ID, c5.concept_name as UNIT_CONCEPT_NAME, a.VISIT_OCCURRENCE_ID, c6.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME,
    OBSERVATION_SOURCE_VALUE, OBSERVATION_SOURCE_CONCEPT_ID, c7.concept_name as SOURCE_CONCEPT_NAME,
    c7.concept_code as SOURCE_CONCEPT_CODE, c7.vocabulary_id as SOURCE_VOCABULARY, UNIT_SOURCE_VALUE,
    QUALIFIER_SOURCE_VALUE, value_source_concept_id, value_source_value, questionnaire_response_id
FROM \`$BQ_PROJECT.$BQ_DATASET.observation\` a
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.OBSERVATION_CONCEPT_ID = c1.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.OBSERVATION_TYPE_CONCEPT_ID = c2.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.value_as_concept_id = c3.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on a.qualifier_concept_id = c4.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.unit_concept_id = c5.CONCEPT_ID
left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.VISIT_OCCURRENCE_ID = v.VISIT_OCCURRENCE_ID
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c6 on v.visit_concept_id = c6.concept_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c7 on a.OBSERVATION_SOURCE_CONCEPT_ID = c7.CONCEPT_ID"

echo "ds_person - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_person\`
    (PERSON_ID, GENDER_CONCEPT_ID, GENDER, DATE_OF_BIRTH, RACE_CONCEPT_ID, RACE, ETHNICITY_CONCEPT_ID, ETHNICITY, SEX_AT_BIRTH_CONCEPT_ID, SEX_AT_BIRTH)
SELECT
    PERSON_ID, GENDER_CONCEPT_ID, c1.concept_name as GENDER, BIRTH_DATETIME as DATE_OF_BIRTH, RACE_CONCEPT_ID,
    c2.concept_name as RACE, ETHNICITY_CONCEPT_ID, c3.concept_name as ETHNICITY, SEX_AT_BIRTH_CONCEPT_ID, c4.concept_name as SEX_AT_BIRTH
FROM \`$BQ_PROJECT.$BQ_DATASET.person\` a
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.gender_concept_id = c1.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.race_concept_id = c2.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.ethnicity_concept_id = c3.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on a.sex_at_birth_concept_id = c4.CONCEPT_ID"

if [[ "$TABLE_LIST" == *'visit_detail'* ]];
then
  echo "ds_procedure_occurrence (OMOP 5.3.1) - inserting data"
  bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
  "INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_procedure_occurrence\`
      (PERSON_ID, PROCEDURE_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY,
      PROCEDURE_DATETIME, PROCEDURE_TYPE_CONCEPT_ID, PROCEDURE_TYPE_CONCEPT_NAME, MODIFIER_CONCEPT_ID, MODIFIER_CONCEPT_NAME,
      QUANTITY, VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, PROCEDURE_SOURCE_VALUE, PROCEDURE_SOURCE_CONCEPT_ID,
      SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, MODIFIER_SOURCE_VALUE)
  select
      a.PERSON_ID, PROCEDURE_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
      c1.vocabulary_id as STANDARD_VOCABULARY, PROCEDURE_DATETIME, PROCEDURE_TYPE_CONCEPT_ID,
      c2.concept_name as PROCEDURE_TYPE_CONCEPT_NAME, MODIFIER_CONCEPT_ID, c3.concept_name as MODIFIER_CONCEPT_NAME,
      QUANTITY, a.VISIT_OCCURRENCE_ID, c4.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME, PROCEDURE_SOURCE_VALUE,
      PROCEDURE_SOURCE_CONCEPT_ID, c5.concept_name as SOURCE_CONCEPT_NAME, c5.concept_code as SOURCE_CONCEPT_CODE,
      c5.vocabulary_id as SOURCE_VOCABULARY, MODIFIER_SOURCE_VALUE
  from \`$BQ_PROJECT.$BQ_DATASET.procedure_occurrence\` a
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.PROCEDURE_CONCEPT_ID = c1.CONCEPT_ID
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.PROCEDURE_TYPE_CONCEPT_ID = c2.CONCEPT_ID
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.MODIFIER_CONCEPT_ID = c3.CONCEPT_ID
  left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.VISIT_OCCURRENCE_ID = v.VISIT_OCCURRENCE_ID
  left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on v.visit_concept_id = c4.concept_id
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.PROCEDURE_SOURCE_CONCEPT_ID = c5.CONCEPT_ID"
else
  echo "ds_procedure_occurrence - inserting data"
  bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
  "INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_procedure_occurrence\`
      (PERSON_ID, PROCEDURE_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY,
      PROCEDURE_DATETIME, PROCEDURE_TYPE_CONCEPT_ID, PROCEDURE_TYPE_CONCEPT_NAME, MODIFIER_CONCEPT_ID, MODIFIER_CONCEPT_NAME,
      QUANTITY, VISIT_OCCURRENCE_ID, VISIT_OCCURRENCE_CONCEPT_NAME, PROCEDURE_SOURCE_VALUE, PROCEDURE_SOURCE_CONCEPT_ID,
      SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, QUALIFIER_SOURCE_VALUE)
  select
      a.PERSON_ID, PROCEDURE_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
      c1.vocabulary_id as STANDARD_VOCABULARY, PROCEDURE_DATETIME, PROCEDURE_TYPE_CONCEPT_ID,
      c2.concept_name as PROCEDURE_TYPE_CONCEPT_NAME, MODIFIER_CONCEPT_ID, c3.concept_name as MODIFIER_CONCEPT_NAME,
      QUANTITY, a.VISIT_OCCURRENCE_ID, c4.concept_name as VISIT_OCCURRENCE_CONCEPT_NAME, PROCEDURE_SOURCE_VALUE,
      PROCEDURE_SOURCE_CONCEPT_ID, c5.concept_name as SOURCE_CONCEPT_NAME, c5.concept_code as SOURCE_CONCEPT_CODE,
      c5.vocabulary_id as SOURCE_VOCABULARY, QUALIFIER_SOURCE_VALUE
  from \`$BQ_PROJECT.$BQ_DATASET.procedure_occurrence\` a
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.PROCEDURE_CONCEPT_ID = c1.CONCEPT_ID
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.PROCEDURE_TYPE_CONCEPT_ID = c2.CONCEPT_ID
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.MODIFIER_CONCEPT_ID = c3.CONCEPT_ID
  left join \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` v on a.VISIT_OCCURRENCE_ID = v.VISIT_OCCURRENCE_ID
  left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on v.visit_concept_id = c4.concept_id
  LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.PROCEDURE_SOURCE_CONCEPT_ID = c5.CONCEPT_ID"
fi

echo "ds_survey - inserting data for all surveys except cope"
bq --quiet --project_id="$BQ_PROJECT" query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_survey\`
   (person_id, survey_datetime, survey, question_concept_id, question, answer_concept_id, answer)
SELECT  a.person_id,
        a.observation_datetime as survey_datetime,
        c.name as survey,
        d.concept_id as question_concept_id,
        d.concept_name as question,
        e.concept_id as answer_concept_id,
        case when a.value_as_number is not null then CAST(a.value_as_number as STRING) else e.concept_name END as answer
FROM \`$BQ_PROJECT.$BQ_DATASET.observation\` a
JOIN
    (
        SELECT *
        FROM \`$BQ_PROJECT.$BQ_DATASET.prep_concept_ancestor\`
        WHERE ancestor_concept_id in
            (
                SELECT concept_id
                FROM \`$BQ_PROJECT.$BQ_DATASET.cb_criteria\`
                WHERE domain_id = 'SURVEY'
                AND parent_id = 0
                AND concept_id NOT IN (1741006, 1333342)
            )
    ) b on a.observation_source_concept_id = b.descendant_concept_id
JOIN \`$BQ_PROJECT.$BQ_DATASET.survey_conduct\` f on a.questionnaire_response_id = f.survey_conduct_id
JOIN \`$BQ_PROJECT.$BQ_DATASET.cb_criteria\` c ON b.ancestor_concept_id = c.concept_id and f.survey_concept_id = c.concept_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` d on a.observation_source_concept_id = d.concept_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` e on a.value_source_concept_id = e.concept_id"

echo "ds_survey - inserting data for all cope surveys"
bq --quiet --project_id="$BQ_PROJECT" query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_survey\`
   (person_id, survey_datetime, survey, question_concept_id, question, answer_concept_id, answer, survey_version_concept_id, survey_version_name)
SELECT  a.person_id,
        a.observation_datetime as survey_datetime,
        case when g.display_name like '%COPE Survey' then 'COVID-19 Participant Experience (COPE) Survey' when g.display_name like '%Minute Survey' then 'COVID-19 Vaccine Survey' else c.name end as survey,
        d.concept_id as question_concept_id,
        d.concept_name as question,
        e.concept_id as answer_concept_id,
        case when a.value_as_number is not null then CAST(a.value_as_number as STRING) else e.concept_name END as answer,
        g.survey_version_concept_id as survey_version_concept_id,
        g.display_name as survey_version_name
FROM \`$BQ_PROJECT.$BQ_DATASET.observation\` a
JOIN
    (
        SELECT *
        FROM \`$BQ_PROJECT.$BQ_DATASET.prep_concept_ancestor\`
        WHERE ancestor_concept_id in
            (
                SELECT concept_id
                FROM \`$BQ_PROJECT.$BQ_DATASET.cb_criteria\`
                WHERE domain_id = 'SURVEY'
                AND parent_id = 0
                AND concept_id IN (1741006, 1333342)
            )
    ) b on a.observation_source_concept_id = b.descendant_concept_id
JOIN \`$BQ_PROJECT.$BQ_DATASET.cb_criteria\` c ON b.ancestor_concept_id = c.concept_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` d on a.observation_source_concept_id = d.concept_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` e on a.value_source_concept_id = e.concept_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.survey_conduct\` f on a.questionnaire_response_id = f.survey_conduct_id
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.cb_survey_version\` g on f.survey_concept_id = g.survey_version_concept_id
GROUP BY a.person_id,
        survey_datetime,
        survey,
        question_concept_id,
        question,
        answer_concept_id,
        answer,
        survey_version_concept_id,
        survey_version_name"

echo "ds_visit_occurrence - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_visit_occurrence\`
    (PERSON_ID, VISIT_CONCEPT_ID, STANDARD_CONCEPT_NAME, STANDARD_CONCEPT_CODE, STANDARD_VOCABULARY, VISIT_START_DATETIME,
    VISIT_END_DATETIME, VISIT_TYPE_CONCEPT_ID, VISIT_TYPE_CONCEPT_NAME, VISIT_SOURCE_VALUE, VISIT_SOURCE_CONCEPT_ID,
    SOURCE_CONCEPT_NAME, SOURCE_CONCEPT_CODE, SOURCE_VOCABULARY, ADMITTING_SOURCE_CONCEPT_ID, ADMITTING_SOURCE_CONCEPT_NAME,
    ADMITTING_SOURCE_VALUE, DISCHARGE_TO_CONCEPT_ID, DISCHARGE_TO_CONCEPT_NAME, DISCHARGE_TO_SOURCE_VALUE)
select
    a.PERSON_ID, VISIT_CONCEPT_ID, c1.concept_name as STANDARD_CONCEPT_NAME, c1.concept_code as STANDARD_CONCEPT_CODE,
    c1.vocabulary_id as STANDARD_VOCABULARY, VISIT_START_DATETIME, VISIT_END_DATETIME, VISIT_TYPE_CONCEPT_ID,
    c2.concept_name as VISIT_TYPE_CONCEPT_NAME, VISIT_SOURCE_VALUE, VISIT_SOURCE_CONCEPT_ID, c3.concept_name as SOURCE_CONCEPT_NAME,
    c3.concept_code as SOURCE_CONCEPT_CODE, c3.vocabulary_id as SOURCE_VOCABULARY, ADMITTING_SOURCE_CONCEPT_ID,
    c4.concept_name as ADMITTING_SOURCE_CONCEPT_NAME, ADMITTING_SOURCE_VALUE, DISCHARGE_TO_CONCEPT_ID,
    c5.concept_name as DISCHARGE_TO_CONCEPT_NAME, DISCHARGE_TO_SOURCE_VALUE
from \`$BQ_PROJECT.$BQ_DATASET.visit_occurrence\` a
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c1 on a.VISIT_CONCEPT_ID = c1.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c2 on a.VISIT_TYPE_CONCEPT_ID = c2.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c3 on a.VISIT_SOURCE_CONCEPT_ID = c3.CONCEPT_ID
left join \`$BQ_PROJECT.$BQ_DATASET.concept\` c4 on a.ADMITTING_SOURCE_CONCEPT_ID = c4.CONCEPT_ID
LEFT JOIN \`$BQ_PROJECT.$BQ_DATASET.concept\` c5 on a.DISCHARGE_TO_CONCEPT_ID = c5.CONCEPT_ID"

echo "ds_activity_summary - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_activity_summary\`
    (date, activity_calories, calories_bmr, calories_out, elevation, fairly_active_minutes, floors,
    lightly_active_minutes, marginal_calories, sedentary_minutes, steps, very_active_minutes, person_id)
SELECT date, activity_calories, calories_bmr, calories_out, elevation, fairly_active_minutes, floors,
    lightly_active_minutes, marginal_calories, sedentary_minutes, steps, very_active_minutes, person_id
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY person_id ) AS rank
FROM \`$BQ_PROJECT.$BQ_DATASET.activity_summary\`)
where rank = 1"

echo "ds_heart_rate_minute_level - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_heart_rate_minute_level\`
    (datetime, person_id, heart_rate_value)
SELECT datetime, person_id, heart_rate_value
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY person_id ) AS rank
FROM \`$BQ_PROJECT.$BQ_DATASET.heart_rate_minute_level\`)
where rank = 1"

echo "ds_heart_rate_summary - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_heart_rate_summary\`
    (person_id, date, zone_name, min_heart_rate, max_heart_rate, minute_in_zone, calorie_count)
SELECT person_id, date, zone_name, min_heart_rate, max_heart_rate, minute_in_zone, calorie_count
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY person_id ) AS rank
FROM \`$BQ_PROJECT.$BQ_DATASET.heart_rate_summary\`)
where rank = 1"

echo "ds_steps_intraday - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_steps_intraday\`
    (datetime, steps, person_id)
SELECT datetime, steps, person_id
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY person_id ) AS rank
FROM \`$BQ_PROJECT.$BQ_DATASET.steps_intraday\`)
where rank = 1"

if [[ "$TABLE_LIST" == *"zip3_ses_map"* ]]; then
  echo "ds_zip_code_socioeconomic - inserting data"
  bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
  "INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_zip_code_socioeconomic\`
  (PERSON_ID, OBSERVATION_DATETIME, ZIP3_AS_STRING, FRACTION_ASSISTED_INCOME, FRACTION_HIGH_SCHOOL_EDU, MEDIAN_INCOME, FRACTION_NO_HEALTH_INS, FRACTION_POVERTY, FRACTION_VACANT_HOUSING, DEPRIVATION_INDEX, ACS)
  select o.person_id,
  o.observation_datetime,
  zip.zip3_as_string,
  zip.fraction_assisted_income,
  zip.fraction_high_school_edu,
  zip.median_income,
  zip.fraction_no_health_ins,
  zip.fraction_poverty,
  zip.fraction_vacant_housing,
  zip.deprivation_index,
  zip.acs
  from \`$BQ_PROJECT.$BQ_DATASET.observation\` o
  join \`$BQ_PROJECT.$BQ_DATASET.zip3_ses_map\` zip on CAST(SUBSTR(o.value_as_string, 0, STRPOS(o.value_as_string, '*') - 1) as INT64) = zip.zip3
  where observation_source_concept_id = 1585250
  and o.value_as_string not like 'Res%'"
fi

echo "ds_sleep_daily_summary - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_sleep_daily_summary\`
    (person_id, sleep_date, is_main_sleep, minute_in_bed, minute_asleep, minute_after_wakeup, minute_awake, minute_restless, minute_deep, minute_light, minute_rem, minute_wake)
SELECT person_id, sleep_date, is_main_sleep, minute_in_bed, minute_asleep, minute_after_wakeup, minute_awake, minute_restless, minute_deep, minute_light, minute_rem, minute_wake
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY person_id ) AS rank
FROM \`$BQ_PROJECT.$BQ_DATASET.sleep_daily_summary\`)
where rank = 1"

echo "ds_sleep_level - inserting data"
bq --quiet --project_id=$BQ_PROJECT query --nouse_legacy_sql \
"INSERT INTO \`$BQ_PROJECT.$BQ_DATASET.ds_sleep_level\`
    (person_id, sleep_date, is_main_sleep, level, start_datetime, duration_in_min)
SELECT person_id, sleep_date, is_main_sleep, level, start_datetime, duration_in_min
FROM (SELECT *, ROW_NUMBER() OVER (PARTITION BY person_id ) AS rank
FROM \`$BQ_PROJECT.$BQ_DATASET.sleep_level\`)
where rank = 1"

redpajama_set_name: RedPajamaGithub


text:
/**
 * DocuSign REST API
 * The DocuSign REST API provides you with a powerful, convenient, and simple Web services API for interacting with DocuSign.
 *
 * OpenAPI spec version: v2.1
 * Contact: devcenter@docusign.com
 *
 * NOTE: This class is auto generated. Do not edit the class manually and submit a new issue instead.
 *
 */

(function(root, factory) {
  if (typeof define === 'function' && define.amd) {
    // AMD. Register as an anonymous module.
    define(['ApiClient', 'model/Checkbox', 'model/Email', 'model/ModelDate', 'model/ModelNumber', 'model/RadioGroup', 'model/SenderCompany', 'model/SenderName', 'model/Ssn', 'model/TabGroup', 'model/Text', 'model/Zip'], factory);
  } else if (typeof module === 'object' && module.exports) {
    // CommonJS-like environments that support module.exports, like Node.
    module.exports = factory(require('../ApiClient'), require('./Checkbox'), require('./Email'), require('./ModelDate'), require('./ModelNumber'), require('./RadioGroup'), require('./SenderCompany'), require('./SenderName'), require('./Ssn'), require('./TabGroup'), require('./Text'), require('./Zip'));
  } else {
    // Browser globals (root is window)
    if (!root.Docusign) {
      root.Docusign = {};
    }
    root.Docusign.PrefillTabs = factory(root.Docusign.ApiClient, root.Docusign.Checkbox, root.Docusign.Email, root.Docusign.ModelDate, root.Docusign.ModelNumber, root.Docusign.RadioGroup, root.Docusign.SenderCompany, root.Docusign.SenderName, root.Docusign.Ssn, root.Docusign.TabGroup, root.Docusign.Text, root.Docusign.Zip);
  }
}(this, function(ApiClient, Checkbox, Email, ModelDate, ModelNumber, RadioGroup, SenderCompany, SenderName, Ssn, TabGroup, Text, Zip) {
  'use strict';

  /**
   * The PrefillTabs model module.
   * @module model/PrefillTabs
   */

  /**
   * Constructs a new PrefillTabs.
   * @alias module:model/PrefillTabs
   * @class
   */
  var exports = function() {
    var _this = this;

  };

  /**
   * Constructs a PrefillTabs from a plain JavaScript object, optionally creating a new instance.
   * Copies all relevant properties from data to obj if supplied or a new instance if not.
   * @param {Object} data The plain JavaScript object bearing properties of interest.
   * @param {module:model/PrefillTabs} obj Optional instance to populate.
   * @return {module:model/PrefillTabs} The populated PrefillTabs instance.
   */
  exports.constructFromObject = function(data, obj) {
    if (data) {
      obj = obj || new exports();

      if (data.hasOwnProperty('checkboxTabs')) {
        obj['checkboxTabs'] = ApiClient.convertToType(data['checkboxTabs'], [Checkbox]);
      }
      if (data.hasOwnProperty('dateTabs')) {
        obj['dateTabs'] = ApiClient.convertToType(data['dateTabs'], [ModelDate]);
      }
      if (data.hasOwnProperty('emailTabs')) {
        obj['emailTabs'] = ApiClient.convertToType(data['emailTabs'], [Email]);
      }
      if (data.hasOwnProperty('numberTabs')) {
        obj['numberTabs'] = ApiClient.convertToType(data['numberTabs'], [ModelNumber]);
      }
      if (data.hasOwnProperty('radioGroupTabs')) {
        obj['radioGroupTabs'] = ApiClient.convertToType(data['radioGroupTabs'], [RadioGroup]);
      }
      if (data.hasOwnProperty('senderCompanyTabs')) {
        obj['senderCompanyTabs'] = ApiClient.convertToType(data['senderCompanyTabs'], [SenderCompany]);
      }
      if (data.hasOwnProperty('senderNameTabs')) {
        obj['senderNameTabs'] = ApiClient.convertToType(data['senderNameTabs'], [SenderName]);
      }
      if (data.hasOwnProperty('ssnTabs')) {
        obj['ssnTabs'] = ApiClient.convertToType(data['ssnTabs'], [Ssn]);
      }
      if (data.hasOwnProperty('tabGroups')) {
        obj['tabGroups'] = ApiClient.convertToType(data['tabGroups'], [TabGroup]);
      }
      if (data.hasOwnProperty('textTabs')) {
        obj['textTabs'] = ApiClient.convertToType(data['textTabs'], [Text]);
      }
      if (data.hasOwnProperty('zipTabs')) {
        obj['zipTabs'] = ApiClient.convertToType(data['zipTabs'], [Zip]);
      }
    }
    return obj;
  }

  /**
   * Specifies a tag on the document in a location where the recipient can select an option.
   * @member {Array.<module:model/Checkbox>} checkboxTabs
   */
  exports.prototype['checkboxTabs'] = undefined;
  /**
   * Specifies a tab on the document where you want the recipient to enter a date. Date tabs are single-line fields that allow date information to be entered in any format. The tooltip for this tab recommends entering the date as MM/DD/YYYY, but this is not enforced. The format entered by the signer is retained.   If you need a particular date format enforced, DocuSign recommends using a Text tab with a Validation Pattern and Validation Message to enforce the format.
   * @member {Array.<module:model/ModelDate>} dateTabs
   */
  exports.prototype['dateTabs'] = undefined;
  /**
   * Specifies a tag on the document where you want the recipient to enter an email. Email tags are single-line fields that accept any characters. The system checks that a valid email format (i.e. xxx@yyy.zzz) is entered in the tag. It uses the same parameters as a Text tab, with the validation message and pattern set for email information.  When getting information that includes this tab type, the original value of the tab when the associated envelope was sent is included in the response.
   * @member {Array.<module:model/Email>} emailTabs
   */
  exports.prototype['emailTabs'] = undefined;
  /**
   * Specifies a tag on the document where you want the recipient to enter a number. It uses the same parameters as a Text tab, with the validation message and pattern set for number information.  When getting information that includes this tab type, the original value of the tab when the associated envelope was sent is included in the response.
   * @member {Array.<module:model/ModelNumber>} numberTabs
   */
  exports.prototype['numberTabs'] = undefined;
  /**
   * Specifies a tag on the document in a location where the recipient can select one option from a group of options using a radio button. The radio buttons do not have to be on the same page in a document.
   * @member {Array.<module:model/RadioGroup>} radioGroupTabs
   */
  exports.prototype['radioGroupTabs'] = undefined;
  /**
   *
   * @member {Array.<module:model/SenderCompany>} senderCompanyTabs
   */
  exports.prototype['senderCompanyTabs'] = undefined;
  /**
   *
   * @member {Array.<module:model/SenderName>} senderNameTabs
   */
  exports.prototype['senderNameTabs'] = undefined;
  /**
   * Specifies a tag on the document where you want the recipient to enter a Social Security Number (SSN). A SSN can be typed with or without dashes. It uses the same parameters as a Text tab, with the validation message and pattern set for SSN information.  When getting information that includes this tab type, the original value of the tab when the associated envelope was sent is included in the response.
   * @member {Array.<module:model/Ssn>} ssnTabs
   */
  exports.prototype['ssnTabs'] = undefined;
  /**
   *
   * @member {Array.<module:model/TabGroup>} tabGroups
   */
  exports.prototype['tabGroups'] = undefined;
  /**
   * Specifies a that that is an adaptable field that allows the recipient to enter different text information.  When getting information that includes this tab type, the original value of the tab when the associated envelope was sent is included in the response.
   * @member {Array.<module:model/Text>} textTabs
   */
  exports.prototype['textTabs'] = undefined;
  /**
   * Specifies a tag on the document where you want the recipient to enter a ZIP code. The ZIP code can be a five numbers or the ZIP+4 format with nine numbers. The zip code can be typed with or without dashes. It uses the same parameters as a Text tab, with the validation message and pattern set for ZIP code information.  When getting information that includes this tab type, the original value of the tab when the associated envelope was sent is included in the response.
   * @member {Array.<module:model/Zip>} zipTabs
   */
  exports.prototype['zipTabs'] = undefined;

  return exports;
}));


text:
require 'cgi'

class Fastly
  class DictionaryItem < Base
    attr_accessor :dictionary_id, :item_key, :item_value, :service_id

    alias_method :key, :item_key
    alias_method :value, :item_value

    # Return the Service object this belongs to
    def service
      @service ||= fetcher.get(Service, service_id)
    end

    # :nodoc:
    def as_hash
      super.delete_if { |var| %w(service_id dictionary_id).include?(var) }
    end

    def self.get_path(service, dictionary_id, item_key, _opts = {})
      "/service/#{service}/dictionary/#{dictionary_id}/item/#{CGI::escape(item_key)}"
    end

    def self.post_path(opts)
      "/service/#{opts[:service_id]}/dictionary/#{opts[:dictionary_id]}/item"
    end

    def self.put_path(obj)
      get_path(obj.service_id, obj.dictionary_id, obj.item_key)
    end

    def self.delete_path(obj)
      put_path(obj)
    end

    def self.list_path(opts = {})
      "/service/#{opts[:service_id]}/dictionary/#{opts[:dictionary_id]}/items"
    end
  end
end


text:
<?php
defined('BASEPATH') OR exit('No direct script access allowed');
header("Access-Control-Allow-Origin: *");
?>

Logueo

		<form id="login-form" class="text-left" method="post" >

						Correo electrónico
						<input type="email" class="form-control" id="username" name="username" placeholder="Correo electrónico" required>

						Contraseña
						<input type="password" class="form-control" id="password" name="password" placeholder="Contraseña" required>

						<input type="checkbox" id="cazatalentos" name="cazatalentos">
						Cazatalentos

				<button type="submit" value="Login" name="entrar" class="login-button" onclick="document.getElementById('info').style.display='block'; "></button>

Olvidé mi contraseña...Da clic aqui

		</form>

Olvidé mi contraseña

	<form id="forgot-password-form" class="text-left" method="post">

Coloque su correo electrónico registrado y se le enviarán instrucciones sobre cómo restablecer su contraseña.

					Correo electrónico
					<input type="email" class="form-control" id="emailperdido" name="emailperdido" placeholder="Correo electrónico" required>

							<input type="checkbox" id="cazatalentos2" name="cazatalentos2">
							Cazatalentos

Recuerda revisar en tu bandeja de Spam.

			<button id="pass" type="submit" class="login-button" onclick="document.getElementById('info').style.display='block'; " ></button>

	</form>


text:
import tensorflow as tf
from absl import flags
import math

FLAGS = flags.FLAGS

def norm_layer():
  if FLAGS.global_bn:
    return tf.keras.layers.experimental.SyncBatchNormalization()
  else:
    return tf.keras.layers.BatchNormalization()

class FCN_backbone(tf.keras.Model):

  def __init__(self,
               hidden_dims,
               use_bn = True,
               use_relu=True):
    super(FCN_backbone, self).__init__()
    self.fcn_layers = []
    for dim in hidden_dims:
      self.fcn_layers.append(tf.keras.layers.Dense(dim))
      if use_bn:
        self.fcn_layers.append(norm_layer())
      if use_relu:
        self.fcn_layers.append(tf.keras.layers.ReLU())
    self.outputs = []

  def call(self, x, training):
    self.outputs = []
    feats = []
    # final_shape = 1
    # for i in range(tf.rank(x)):
    #   if i > 0:
    #     final_shape *= tf.shape(x)[i]
    # x = tf.reshape(x, (tf.shape(x)[0], final_shape))
    #tf.print('x shape:', tf.shape(x))
    x = tf.reshape(x, (tf.shape(x)[0], -1))
    for ind, layer in enumerate(self.fcn_layers):
      x = layer(x, training=training)
      if ind%3 == 0 and ((ind//3 in FLAGS.HSIC_feature_layers) or FLAGS.use_all_features_HSIC):
        self.outputs.append(x)
        feats.append(x)

    return x, feats

class CNN_backbone(tf.keras.Model):

  def __init__(self,
               filters,
               kernel_sizes,
               strides,
               use_bn = True,
               use_relu=True):
    super(CNN_backbone, self).__init__()
    self.cnn_layers = []
    for ind, filter in enumerate(filters):
      self.cnn_layers.append(tf.keras.layers.Conv2D(filter, kernel_sizes[ind], strides[ind], padding='same'))
      if use_bn:
        self.cnn_layers.append(norm_layer())
      if use_relu:
        self.cnn_layers.append(tf.keras.layers.ReLU())
    self.output_bf_avg_pool = None

  def call(self, x, training):
    feats = []
    for ind, layer in enumerate(self.cnn_layers):
      x = layer(x, training=training)
      if ind%3 == 0 and ((ind//3 in FLAGS.HSIC_feature_layers) or FLAGS.use_all_features_HSIC):
        if FLAGS.use_GAP_HSIC_features:
          feats.append(tf.reduce_mean(x, [1, 2]))
        elif FLAGS.use_random_projections:
          temp_input = tf.reshape(x, (tf.shape(x)[0], -1))
          final_shape = tf.constant([FLAGS.random_proj_dim], dtype=tf.int32)
          final_shape = tf.concat([final_shape, tf.shape(x)[1:]], axis=0)
          z = tf.random.normal(final_shape)
          z = tf.reshape(z, (FLAGS.random_proj_dim, -1))
          z = tf.transpose(z)
          q, _ = tf.linalg.qr(z)
          #z = z/tf.norm(z, axis=1, keepdims=True)
          #feats.append(tf.matmul(temp_input, tf.transpose(z)))
          feats.append(tf.matmul(temp_input, q))
        else:
          feats.append(x)

    self.output_bf_avg_pool = x
    x = tf.reduce_mean(x, axis=[1,2])
    return x, feats

class multi_base_model(tf.keras.Model):

  def __init__(self,
               base_models,
               num_classes):
    super(multi_base_model, self).__init__()
    self.base_models = base_models
    self.out = tf.keras.layers.Dense(num_classes)

  def call(self, x, training, only_head=False, only_linear_head=False, init=False, project_out_w=False, project_out_mat=None,
           gauss_noise_feats=False, sigma=1.0, rand_sigma=False):
    outs = []
    for model_ind, model in enumerate(self.base_models):
      if FLAGS.use_pretrained:
        if gauss_noise_feats:
          out = model.layers[0](x, training = training and not only_head)
          if model_ind == 0:
            if rand_sigma:
              std_devs = sigma * tf.random.uniform([tf.shape(out)[0]])
              final_shape = tf.constant([-1], dtype=tf.int32)
              final_shape = tf.concat([final_shape, tf.ones([tf.rank(out)-1], dtype=tf.int32)], axis=0)
              z = tf.reshape(std_devs, final_shape) * tf.random.normal(tf.shape(out))
            else:
              final_shape = tf.constant([-1], dtype=tf.int32)
              final_shape = tf.concat([final_shape, tf.ones([tf.rank(out)-1], dtype=tf.int32)], axis=0)
              z = sigma * tf.random.normal(tf.shape(out))
          out = out + tf.cast(z, out.dtype)
          for ind, layer in enumerate(model.layers):
            if ind > 0:
              out = layer(out, training = training and not only_head)
        else:
          out = model(x, training = training and not only_head)
        feat = [out]
      else:
        out, feat = model(x, training = training and not only_head)
      outs.append(out)

    inp = tf.concat(outs, axis=1)
    fin = self.out(inp)
    return inp, fin, feat

class base_multi_head_model(tf.keras.Model):

  def __init__(self,
               base_model,
               head_models,
               num_classes):
    super(base_multi_head_model, self).__init__()
    self.base_model = base_model
    self.head_models = head_models
    self.out = tf.keras.layers.Dense(num_classes)

  def call(self, x, training, only_head=False, only_linear_head=False, init=False, project_out_w=False, project_out_mat=None,
           gauss_noise_feats=False, sigma=1.0, rand_sigma=False):
    outs = []
    base_out = self.base_model(x, training = training and not only_head)
    if FLAGS.use_pretrained:
      if gauss_noise_feats:
        if rand_sigma:
          std_devs = sigma * tf.random.uniform([tf.shape(base_out)[0]])
          final_shape = tf.constant([-1], dtype=tf.int32)
          final_shape = tf.concat([final_shape, tf.ones([tf.rank(base_out)-1], dtype=tf.int32)], axis=0)
          z = tf.reshape(std_devs, final_shape) * tf.random.normal(tf.shape(base_out))
        else:
          final_shape = tf.constant([-1], dtype=tf.int32)
          final_shape = tf.concat([final_shape, tf.ones([tf.rank(base_out)-1], dtype=tf.int32)], axis=0)
          z = sigma * tf.random.normal(tf.shape(base_out))
        base_out = base_out + z
    for model_ind, model in enumerate(self.head_models):
      out = model(base_out, training = training and not only_head)
      feat = [out]
      outs.append(out)

    inp = tf.concat(outs, axis=1)
    fin = self.out(inp)
    return inp, fin, feat

class head_model(tf.keras.Model):

  def __init__(self,
               base_model,
               num_classes,
               proj_dim,
               proj_layer,
               head_dims,
               use_proj=True,
               use_bn=True,
               use_relu=True,
               dropout_rate=0.0):
    super(head_model, self).__init__()
    self.base_model = base_model
    if proj_layer > len(head_dims):
      raise ValueError('proj layer must be less than head dimensions')
    self.proj_head = []
    for i in range(proj_layer):
      self.proj_head.append(tf.keras.layers.Dense(head_dims[i]))
      if use_bn:
        self.proj_head.append(norm_layer())
      if use_relu:
        self.proj_head.append(tf.keras.layers.ReLU())

    if use_proj:
      self.proj_head.append(tf.keras.layers.Dense(proj_dim))

    self.head = []
    self.head.append(tf.keras.layers.Dropout(rate=dropout_rate))
    for i in range(len(head_dims) - proj_layer):
      self.head.append(tf.keras.layers.Dense(head_dims[proj_layer + i]))
      if use_bn:
        self.head.append(norm_layer())
      if use_relu:
        self.head.append(tf.keras.layers.ReLU())

    if FLAGS.use_chizat_init:
      if len(head_dims) > 1 or num_classes > 1:
        raise ValueError('Rich regime initialization only for 1 layer net and binary classification')
      else:
        self.out = tf.keras.layers.Dense(num_classes, use_bias=False)
    else:
      self.out = tf.keras.layers.Dense(num_classes)

    self.head_dims = head_dims

  def call(self, x, training, only_head=False, only_linear_head=False, init=False, project_out_w=False, project_out_mat=None,
           gauss_noise_feats=False, sigma=1.0, rand_sigma=False):
    if FLAGS.use_pretrained:
      x = self.base_model(x, training=training and not only_head)
      if gauss_noise_feats:
        if rand_sigma:
          std_devs = sigma * tf.random.uniform([tf.shape(x)[0]])
          final_shape = tf.constant([-1], dtype=tf.int32)
          final_shape = tf.concat([final_shape, tf.ones([tf.rank(x)-1], dtype=tf.int32)], axis=0)
          z = tf.reshape(std_devs, final_shape) * tf.random.normal(tf.shape(x))
        else:
          z = sigma * tf.random.normal(tf.shape(x))
        x = x + tf.cast(z, x.dtype)
      feat = []
    else:
      x, feat = self.base_model(x, training=training and not only_head)
    for layer in self.proj_head:
      x = layer(x, training=training and not only_linear_head)
    reps = x
    if project_out_w and FLAGS.use_pretrained:
      for ind, layer in enumerate(self.head):
        if isinstance(layer, tf.keras.layers.Dense):
          for var in layer.trainable_variables:
            if 'kernel' in var.name:
              W_norm = tf.norm(var, axis=0, keepdims=True)**2
              sample_ind = tf.random.categorical(tf.math.log(W_norm), 1)[0,0]
              sample_W = tf.reshape(var[:, sample_ind], [-1,1])
              ctx = tf.distribute.get_replica_context()
              sample_W_gather = ctx.all_gather(sample_W, axis=1)
              sample_W = tf.reshape(sample_W_gather[:,0], [-1,1])
              sample_W = sample_W/tf.norm(sample_W, axis=0, keepdims=True)
              proj_mat = tf.eye(tf.shape(var)[0]) - tf.linalg.matmul(sample_W, tf.transpose(sample_W))
              x = tf.transpose(tf.linalg.matmul(proj_mat, tf.transpose(x)))
          break
    elif project_out_mat is not None:
      pinv = tf.linalg.pinv(tf.linalg.matmul(tf.transpose(project_out_mat), project_out_mat))
      proj_mat = tf.linalg.matmul(project_out_mat, tf.linalg.matmul(pinv, tf.transpose(project_out_mat)))
      proj_x = tf.linalg.matmul(proj_mat, tf.transpose(x))
      x = x - tf.transpose(proj_x)
    curr_ind = 0
    for layer in self.head:
      if FLAGS.use_chizat_init and isinstance(layer, tf.keras.layers.Dense):
        if init:
          tf.print('assigned')
          z = tf.random.normal((tf.shape(x)[1]+1, self.head_dims[0]))
          z = z/tf.norm(z, axis=0, keepdims=True)
          layer.kernel.assign(z[0:tf.shape(x)[1], :])
          layer.bias.assign(z[tf.shape(x)[1], :])
      x = layer(x, training=training and not only_linear_head)
      if FLAGS.use_pretrained:
        if isinstance(layer, tf.keras.layers.Dense):
          if curr_ind in FLAGS.HSIC_feature_layers or FLAGS.use_all_features_HSIC:
            feat.append(x)
          curr_ind += 1
    if FLAGS.use_chizat_init:
      if init:
        tf.print('out assigned')
        samples = tf.random.categorical(tf.math.log([[0.5, 0.5]]), tf.shape(x)[1])
        self.out.kernel.assign(tf.transpose(2.0*tf.cast(samples, tf.float32) - 1.0))
    x = self.out(x, training=training)

    if FLAGS.use_chizat_init:
      x = x/tf.cast(self.head_dims[0], tf.float32)
    if FLAGS.use_all_features_HSIC or -1 in FLAGS.HSIC_feature_layers:
      feat.append(x)
    return reps, x, feat

class multihead_model(tf.keras.Model):

  def __init__(self,
               base_model,
               num_classes,
               proj_dim,
               proj_layer,
               head_dims,
               num_heads,
               use_proj=True,
               use_bn=True,
               use_relu=True,
               dropout_rate=0.0,
               normalized_reps=False):
    super(multihead_model, self).__init__()
    self.base_model = base_model
    self.proj_heads = []
    self.heads = []
    self.outs = []
    self.num_heads = num_heads
    self.normalized_reps = normalized_reps

    for i in range(num_heads):
      curr_proj_head = []
      for j in range(proj_layer):
        curr_proj_head.append(tf.keras.layers.Dense(head_dims[j]))
        if use_bn:
          curr_proj_head.append(norm_layer())
        if use_relu:
          curr_proj_head.append(tf.keras.layers.ReLU())
      if use_proj:
        curr_proj_head.append(tf.keras.layers.Dense(proj_dim))
      self.proj_heads.append(tf.keras.Sequential(layers = curr_proj_head))
      curr_head = []
      curr_head.append(tf.keras.layers.Dropout(rate=dropout_rate))
      for j in range(len(head_dims) - proj_layer):
        curr_head.append(tf.keras.layers.Dense(head_dims[j + proj_layer]))
        if use_bn:
          curr_head.append(norm_layer())
        if use_relu:
          curr_head.append(tf.keras.layers.ReLU())
      self.heads.append(tf.keras.Sequential(layers = curr_head))
      self.outs.append(tf.keras.layers.Dense(num_classes))

  def call(self, x, training, only_head=False, only_linear_head=False):
    x = self.base_model(x, training=training and not only_head)

    outs = []
    reps = []
    for i in range(self.num_heads):
      y = self.proj_heads[i](x, training=training and not only_linear_head)
      if self.normalized_reps:
        y = y/tf.norm(y, axis=1, keepdims=True)
      reps.append(y)
      y = self.heads[i](y, training=training and not only_linear_head)
      y = self.outs[i](y, training = training)
      outs.append(y)
    return reps, outs

class WarmUpAndCosineDecay(tf.keras.optimizers.schedules.LearningRateSchedule):
  """Applies a warmup schedule on a given learning rate decay schedule."""

  def __init__(self, base_learning_rate, num_examples, name=None):
    super(WarmUpAndCosineDecay, self).__init__()
    self.base_learning_rate = base_learning_rate
    self.num_examples = num_examples
    self._name = name

  def __call__(self, step):
    with tf.name_scope(self._name or 'WarmUpAndCosineDecay'):
      steps_per_epoch = (self.num_examples // FLAGS.train_batch_size) + 1
      if FLAGS.warmup_steps is not None:
        warmup_steps = FLAGS.warmup_steps
      else:
        warmup_steps = int(FLAGS.warmup_epochs * steps_per_epoch)
      if FLAGS.learning_rate_scaling == 'linear':
        scaled_lr = self.base_learning_rate * FLAGS.train_batch_size / 256.
      elif FLAGS.learning_rate_scaling == 'sqrt':
        scaled_lr = self.base_learning_rate * math.sqrt(FLAGS.train_batch_size)
      else:
        raise ValueError('Unknown learning rate scaling {}'.format(
            FLAGS.learning_rate_scaling))
      learning_rate = (
          step / float(warmup_steps) * scaled_lr if warmup_steps else scaled_lr)

      # Cosine decay learning rate schedule
      if FLAGS.train_steps > 0:
        total_steps = FLAGS.train_steps
      else:
        total_steps = int(FLAGS.train_epochs * steps_per_epoch)

      cosine_decay = tf.keras.experimental.CosineDecay(
          scaled_lr, total_steps - warmup_steps)
      learning_rate = tf.where(step < warmup_steps, learning_rate,
                               cosine_decay(step - warmup_steps))

      return learning_rate

  def get_config(self):
    return {
        'base_learning_rate': self.base_learning_rate,
        'num_examples': self.num_examples,
    }


text:
package com.github.projectflink

import java.lang
import java.lang.management.ManagementFactory

import org.apache.flink.api.common.accumulators.LongCounter
import org.apache.flink.api.common.functions.RichMapPartitionFunction
import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint
import org.apache.flink.api.scala._
import org.apache.flink.configuration.Configuration
import org.apache.flink.core.fs.FileSystem.WriteMode
import org.apache.flink.util.Collector

import scala.collection.mutable

import scala.collection.JavaConverters._
import scala.util.Random

class MTuple3[T1, T2, T3](var _1: T1, var _2: T2, var _3: T3) {
  def this() = this(null.asInstanceOf[T1], null.asInstanceOf[T2], null.asInstanceOf[T3])

  override def hashCode = 73*_1.hashCode() + 79*_2.hashCode() + _3.hashCode()
  override def equals(other: Any): Boolean = {
    if (!(other.isInstanceOf[MTuple3[_, _, _]])) {
      return false
    }
    val oTuple = other.asInstanceOf[MTuple3[_, _, _]]
    _1.equals(oTuple._1) && _2.equals(oTuple._2) && _3.equals(oTuple._3)
  }

  override def toString = "M(" + _1 + "," + _2 + "," + _3 + ")"
}

class MTuple2[T1, T2](var _1: T1, var _2: T2) {
  def this() = this(null.asInstanceOf[T1], null.asInstanceOf[T2])

  override def hashCode = 73*_1.hashCode() + _2.hashCode()
  override def equals(other: Any): Boolean = {
    if (!(other.isInstanceOf[MTuple2[_, _]])) {
      return false
    }
    val oTuple = other.asInstanceOf[MTuple2[_, _]]
    _1.equals(oTuple._1) && _2.equals(oTuple._2)
  }

  override def toString = "M(" + _1 + "," + _2 + ")"
}

class MTuple2Cheat[T1, T2](var _1: T1, var _2: T2) {
  def this() = this(null.asInstanceOf[T1], null.asInstanceOf[T2])

  override def hashCode = _1.hashCode()
  override def equals(other: Any): Boolean = {
    if (!(other.isInstanceOf[MTuple2Cheat[_, _]])) {
      return false
    }
    val oTuple = other.asInstanceOf[MTuple2Cheat[_, _]]
    _1.equals(oTuple._1)
  }

  override def toString = "M(" + _1 + "," + _2 + ")"
}

object GroupReduceBenchmarkGenerateData {
  val rnd = new Random(System.currentTimeMillis())

  private final def skewedSample(skew: Double, max: Long): Long = {
    val uniform = rnd.nextDouble()
    val `var` = Math.pow(uniform, skew)
    val pareto = 0.2 / `var`
    val `val` = pareto.toInt
    if (`val` > max) `val` % max else `val`
  }

  def main(args: Array[String]) {
    if (args.length < 6) {
      println("Usage: [master] [dop] [numCountries] [numBooks] [numReads] [skew] [output]")
      return
    }

    var master = args(0)
    var dop = args(1).toInt
    var numCountries = args(2).toLong
    var numBooks = args(3).toInt
    var numReads = args(4).toLong
    var skew = args(5).toFloat
    val outputPath = if (args.length > 6) {
      args(6)
    } else {
      null
    }

    // set up the execution environment
    val env = ExecutionEnvironment.getExecutionEnvironment
    env.setDegreeOfParallelism(dop)

    val countryIds = env.generateSequence(0, numCountries - 1)
    val bookIds = env.generateSequence(0, numBooks - 1)

    val countryNames = countryIds map { id => (id, "No. " + id + " The awesome country") }
    val bookNames = bookIds map { id => (id, "No. " + id + " This is the most special song") }

    val reads = env.generateSequence(0, numReads - 1) map {
      i =>
        (skewedSample(skew, numCountries - 1), rnd.nextInt(numBooks).toLong)
    }

    val readsWithCountry = reads.joinWithTiny(countryNames).where("_1").equalTo("_1") {
      (left, right) =>
        (right._2, left._2)
    }

    val readsWithCountryAndBook = readsWithCountry.join(bookNames, JoinHint.REPARTITION_HASH_SECOND)
      .where("_2")
      .equalTo("_1") {
      (left, right) =>
        (left._1, right._2)
    }.rebalance()

    if (outputPath == null) {
      readsWithCountryAndBook.print()
    } else {
      readsWithCountryAndBook.writeAsCsv(outputPath + s"$numCountries-$numBooks-$numReads-$skew",
        rowDelimiter = "\n",
        fieldDelimiter = ",",
        writeMode = WriteMode.OVERWRITE)
    }

    // execute program
    env.execute("Group Reduce Benchmark Data Generator")
  }
}

object GroupReduceBenchmarkFlink {

  def main(args: Array[String]) {
    if (args.length < 4) {
      println("Usage: [master] [dop] [k] [input] [output]")
      return
    }

    var master = args(0)
    var dop = args(1).toInt
    var k = args(2).toInt
    var inputPath = args(3)
    val outputPath = if (args.length > 4) {
      args(4)
    } else {
      null
    }

    // set up the execution environment
    val env = ExecutionEnvironment.getExecutionEnvironment
    env.setDegreeOfParallelism(dop)

    val readsWithCountryAndBook = env.readCsvFile[(String, String)](inputPath)

    // Group by country and determine top-k books
    val result = readsWithCountryAndBook
      .map { in => new MTuple2(new MTuple2(in._1, in._2), 1L) }
      .groupBy("_1.*")
      .sum("_2")
//      .groupBy("_1").sortGroup("_3", Order.DESCENDING)
//      .first(k)
//      .groupBy("_1")
//      .reduceGroup {
//      in =>
//        val it = in.toIterator.buffered
//        val first = it.head
//        (first._1, it.map(in => (in._2, in._3)).mkString(", "))
//    }

    if (outputPath == null) {
      result.print()
    } else {
      result.writeAsCsv(outputPath + "_flink", writeMode = WriteMode.OVERWRITE)
    }

    // execute program
    env.execute("Group Reduce Benchmark Flink")
//    println(env.getExecutionPlan())
  }
}

object GroupReduceBenchmarkFlinkHashCombine {

  def main(args: Array[String]) {
    if (args.length < 4) {
      println("Usage: [master] [dop] [k] [input] [output]")
      return
    }

    var master = args(0)
    var dop = args(1).toInt
    var k = args(2).toInt
    var inputPath = args(3)
    val outputPath = if (args.length > 4) {
      args(4)
    } else {
      null
    }

    // set up the execution environment
    val env = ExecutionEnvironment.getExecutionEnvironment
    env.setDegreeOfParallelism(dop)

    val readsWithCountryAndBook = env.readCsvFile[(String, String)](inputPath)

    // Group by country and determine top-k books
    val result = readsWithCountryAndBook
      .map { in => new MTuple2Cheat(new MTuple2(in._1, in._2), 1L) }
      .mapPartition(new MutableHashAggregator("COMBINER"))
      .partitionByHash("_1.*")
      .groupBy("_1.*")
      .sum("_2")
//      .groupBy("_1").sortGroup("_3", Order.DESCENDING)
//      .first(k)
//      .groupBy("_1")
//      .reduceGroup {
//      in =>
//        val it = in.toIterator.buffered
//        val first = it.head
//        (first._1, it.map(in => (in._2, in._3)).mkString(", "))
//    }

    if (outputPath == null) {
      result.print()
    } else {
      result.writeAsCsv(outputPath + "_flink", writeMode = WriteMode.OVERWRITE)
    }

    // execute program
    env.execute("Group Reduce Benchmark Flink (HashCombine)")
    //        println(env.getExecutionPlan())
  }
}

object GroupReduceBenchmarkFlinkHash {

  def main(args: Array[String]) {
    if (args.length < 4) {
      println("Usage: [master] [dop] [k] [input] [output]")
      return
    }

    var master = args(0)
    var dop = args(1).toInt
    var k = args(2).toInt
    var inputPath = args(3)
    val outputPath = if (args.length > 4) {
      args(4)
    } else {
      null
    }

    // set up the execution environment
    val env = ExecutionEnvironment.getExecutionEnvironment
    env.setDegreeOfParallelism(dop)

    val readsWithCountryAndBook = env.readCsvFile[(String, String)](inputPath)
//    val readsWithCountryAndBook = env.fromElements(
//      ("Land 1", "Song 1"),
//      ("Land 1", "Song 1"),
//      ("Land 1", "Song 1"),
//      ("Land 3", "Song 3"),
//      ("Land 3", "Song 3"),
//      ("Land 2", "Song 2"))

    // Group by country and determine top-k books
    val result = readsWithCountryAndBook
      .map { in => new MTuple2Cheat(new MTuple2(in._1, in._2), 1L) }
      .mapPartition(new MutableHashAggregator("COMBINER"))
      .partitionByHash("_1.*")
      .mapPartition(new MutableHashAggregator("REDUCER"))
      .map { in => (in._1._1, in._1._2, in._2) }
//      .groupBy("_1").sortGroup("_3", Order.DESCENDING)
//      .first(k)
//      .groupBy("_1")
//      .reduceGroup {
//      in =>
//        val it = in.toIterator.buffered
//        val first = it.head
//        (first._1, it.map(in => (in._2, in._3)).mkString(", "))
//    }

    if (outputPath == null) {
      result.print()
    } else {
      result.writeAsCsv(outputPath + "_flink", writeMode = WriteMode.OVERWRITE)
    }

    // execute program
    env.execute("Group Reduce Benchmark Flink (Hash)")
//        println(env.getExecutionPlan())
  }
}

class MutableHashAggregator(combine: String)
  extends RichMapPartitionFunction[
    MTuple2Cheat[MTuple2[String, String], Long],
    MTuple2Cheat[MTuple2[String, String], Long]] {

  private var aggregate: mutable.HashMap[
    MTuple2Cheat[MTuple2[String, String], Long],
    MTuple2Cheat[MTuple2[String, String], Long]] = null

  var gcCount = new LongCounter
  var gcTime = new LongCounter

  override def open(config: Configuration) {
    aggregate = mutable.HashMap[
      MTuple2Cheat[MTuple2[String, String], Long],
      MTuple2Cheat[MTuple2[String, String], Long]]()
    val ctx = getRuntimeContext
    ctx.addAccumulator("gc-time-" + ctx.getIndexOfThisSubtask, gcTime)
    ctx.addAccumulator("gc-count-" + ctx.getIndexOfThisSubtask, gcCount)
  }

  override def mapPartition(
      records: lang.Iterable[MTuple2Cheat[MTuple2[String, String], Long]],
      out: Collector[MTuple2Cheat[MTuple2[String, String], Long]]): Unit = {

    val gcBeans = ManagementFactory.getGarbageCollectorMXBeans.asScala
    var beforeGcCount = 0L
    var beforeGcTime = 0L
    gcBeans foreach {
      gc =>
        beforeGcCount += gc.getCollectionCount
        beforeGcTime += gc.getCollectionTime
    }

    val it = records.iterator
    while (it.hasNext) {
      val value = it.next()
      val agg = aggregate.getOrElse(
        value,
        new MTuple2Cheat(new MTuple2(value._1._1, value._1._2), 0L))
      agg._2 += value._2
      aggregate.put(agg, agg)
//      val newKey = new MTuple2(value._1._1, value._1._2)
    }

    var afterGcCount = 0L
    var afterGcTime = 0L
    gcBeans foreach {
      gc =>
        afterGcCount += gc.getCollectionCount
        afterGcTime += gc.getCollectionTime
    }

    gcCount.add(afterGcCount - beforeGcCount)
    gcTime.add(afterGcTime - beforeGcTime)

    val result = new MTuple2Cheat(new MTuple2("", ""), 0L)
    val outIt = aggregate.values.iterator

    while (outIt.hasNext) {
      val value = outIt.next()
      out.collect(value)
    }
  }
}

//class HashGrouper(k: Int)
//  extends RichMapPartitionFunction[(String, String, Long), MTuple2[String, String]] {
//
//  private var groups: mutable.HashMap[String, mutable.MutableList[(String, Long)]] = null
//
//  override def open(config: Configuration) {
//    groups = mutable.HashMap[String, mutable.MutableList[(String, Long)]]()
//  }
//
//  override def mapPartition(
//      records: lang.Iterable[(String, String, Long)],
//      out: Collector[MTuple2[String, String]]): Unit = {
//    val it = records.iterator()
//    while (it.hasNext) {
//      val value = it.next()
//      val list = groups.getOrElse(value._1, mutable.MutableList[(String, Long)]())
//      list += ((value._2, value._3))
//      groups.put(value._1, list)
//    }
//
//    val listIt = groups.iterator
//    while (listIt.hasNext) {
//      val value = listIt.next()
//      val key = value._1
//      val list = value._2
//      val sorted = list.sortBy(- _._2)
////      println("SORTED " + sorted.take(5))
//      out.collect((key, sorted.take(k).mkString(", ") ))
//    }
//  }
//}


text:
@import MaterialComponents;

NS_ASSUME_NONNULL_BEGIN

@protocol AuthenticationButtonDelegate

- (void) onAuthenticationButtonTapped:(id) sender;

@end

@interface AuthenticationButton : UIView

@property (strong, nonatomic) NSDictionary *strategy;
@property (weak, nonatomic) id<AuthenticationButtonDelegate> delegate;

- (void) applyThemeWithContainerScheme:(id<MDCContainerScheming>)containerScheme;

@end

NS_ASSUME_NONNULL_END


text:
"""Quantized Attention Layers."""

import string
from typing import Tuple, Any, Sequence

from jax import numpy as jnp
from jax.ad_checkpoint import checkpoint_name
from praxis import base_layer
from praxis import pytypes
from praxis.layers import attentions
from praxis.layers.quantization import operations

QuantizationHParams = base_layer.QuantizationHParams
WeightInit = base_layer.WeightInit
WeightHParams = base_layer.WeightHParams
sub_config_field = base_layer.sub_config_field
JTensor = pytypes.JTensor
NestedJTensor = pytypes.NestedJTensor

class AttentionProjection(attentions.AttentionProjection):
  """Layer that computes quantized multi heads projection.

    This layer is expected to be used within DotProductAttention.
  """

  class HParams(attentions.AttentionProjection.HParams):
    """Associated hyper-params for this layer class.

    Attributes:
      quantization: Information related to the quantization applied to this
        layer, such as dtype for the quantized weight.
    """
    quantization: QuantizationHParams = sub_config_field(QuantizationHParams)

  def setup(self) -> None:
    p = self.hparams
    wp = p.weight_split_dims_mapping
    has_sharding = p.mesh_shape is not None and wp.wt is not None
    if p.attention_combine_dims:
      assert not p.use_bias
      hd_shape = [p.num_heads * p.dim_per_head]
    else:
      hd_shape = [p.num_heads, p.dim_per_head]

    if p.attention_combine_dims and has_sharding:
      if len(wp.wt) == 3:
        h_sharding = ()
        for axes in (wp.wt[0], wp.wt[1]):
          if isinstance(axes, (str, int)):
            h_sharding += (axes,)
          elif axes is not None:
            h_sharding += axes
        wt = [h_sharding, wp.wt[2]]
      assert len(wt) == 2
    else:
      wt = wp.wt
    pc_shape = [p.input_dim] + hd_shape
    if p.is_output_projection and p.use_nhd_shape:
      pc_shape = hd_shape + [p.input_dim]
    pc = WeightHParams(
        shape=pc_shape, mesh_shape=p.mesh_shape, tensor_split_dims_mapping=wt)
    if p.quantization.mode == base_layer.QuantizationMode.INFERENCE:
      if p.is_output_projection:
        self.create_quantized_variable('w', pc, [p.input_dim])
      else:
        self.create_quantized_variable('w', pc, hd_shape)
    else:
      self.create_variable('w', pc)
    if p.use_bias:
      if p.is_output_projection:
        if has_sharding:
          bias_split_dims_mapping = [wp.wt[0]]
        else:
          bias_split_dims_mapping = None
        pc_bias = WeightHParams(
            shape=[p.input_dim],
            init=WeightInit.Constant(0.0),
            mesh_shape=p.mesh_shape,
            tensor_split_dims_mapping=bias_split_dims_mapping)
      else:
        if has_sharding:
          bias_split_dims_mapping = [wp.wt[1], wp.wt[2]]
        else:
          bias_split_dims_mapping = None
        pc_bias = WeightHParams(
            shape=[p.num_heads, p.dim_per_head],
            init=WeightInit.Constant(0.0),
            mesh_shape=p.mesh_shape,
            tensor_split_dims_mapping=bias_split_dims_mapping)
      self.create_variable('b', pc_bias)

  def __call__(self, inputs: JTensor) -> JTensor:
    """Computes the multi headed projection for inputs.

    Args:
      inputs: A JTensor of shape [..., num_heads, dim_per_head] if
        p.is_output_projection is True or [..., p.input_dim] otherwise..

    Returns:
      The projected JTensor with shape [..., p.input_dim] if
      p.is_output_projection is True or [..., num_heads, dim_per_head]
      otherwise.
    """
    p = self.hparams
    theta = self.theta

    # Because tf.einsum is not fully optimized unless all the dimensions are
    # fully specified, we have to avoid using '...' for batch dimensions in the
    # equation in tf.einsum for optimized performance. This is only feasible
    # when the rank of the tensor is known.
    # Sort the available symbols to avoid nondeterminism.
    eqn_sym = ''.join(sorted(set(string.ascii_uppercase) - set('DHN')))
    shape = inputs.shape
    rank = len(shape)

    inputs = self._cast_to_fprop_dtype(inputs)
    if p.attention_combine_dims:
      pc_shape = [p.input_dim, p.num_heads, p.dim_per_head]
      if p.is_output_projection and p.use_nhd_shape:
        pc_shape = [p.num_heads, p.dim_per_head, p.input_dim]
      w = jnp.reshape(theta.w, pc_shape)
    else:
      w = theta.w

    if p.is_output_projection:
      assert shape[-2:] == (p.num_heads, p.dim_per_head)
      batch_eqn = eqn_sym[:(rank - 2)]
      if p.use_nhd_shape:
        eqn = f'{batch_eqn}NH,NHD->{batch_eqn}D'
      else:
        eqn = f'{batch_eqn}NH,DNH->{batch_eqn}D'
    else:
      assert shape[-1] == p.input_dim, (
          f'Expecting shape[-1] == p.input_dim, {shape[-1]} != {p.input_dim}')
      batch_eqn = eqn_sym[:(rank - 1)] if rank else '...'
      eqn = f'{batch_eqn}D,DNH->{batch_eqn}NH'

    if p.quantization.mode == base_layer.QuantizationMode.INFERENCE:
      w, s = self.get_quantized_weight('w')
      ret = operations.einsum(eqn, inputs, w, s)
    else:
      ret = jnp.einsum(eqn, inputs, w)
    if p.use_bias:
      ret += theta.b
    return ret

  def quantized_partitioned_specs(self) -> Any:
    """Get quantized PartitionSpec.

    Returns:
      a map from names to partition spec.
    """
    p = self.hparams
    scale_name = 'w' + base_layer.QUANTIZED_NAME_POSTFIX
    weight_pspec = base_layer._weight_hparam_to_pspec(
        self._weight_hparams['w'], self.hparams.mesh_axis_names)
    wp = p.weight_split_dims_mapping
    if p.is_output_projection:
      scale_split_dims_mapping = [wp.wt[0]]
    else:
      scale_split_dims_mapping = [wp.wt[1], wp.wt[2]]
    # scale_weight_hparam is unmaterialized so shape is irrelevant.
    scale_weight_hparam = WeightHParams(
        shape=(), tensor_split_dims_mapping=scale_split_dims_mapping)
    scale_pspec = base_layer._weight_hparam_to_pspec(
        scale_weight_hparam, self.hparams.mesh_axis_names)
    partitionspec = {'w': weight_pspec, scale_name: scale_pspec}
    return {base_layer.PARAMS: partitionspec}

  def quantize_weight(self) -> NestedJTensor:
    """Get quantized weight.

    Returns:
      a map from names to quantized weights.
    """
    p = self.hparams
    assert p.quantization.mode == base_layer.QuantizationMode.QUANTIZE
    eqn = ''
    # This matches the equantion logic in __call__ for weights.
    if p.is_output_projection:
      if p.use_nhd_shape:
        eqn = 'ANH,NHD->AD'
      else:
        eqn = 'ANH,DNH->AD'
    else:
      eqn = 'AD,DNH->ANH'
    q_w, q_s = operations.reduce_einsum_weight_precision(eqn, self.theta.w)
    scale_name = 'w' + base_layer.QUANTIZED_NAME_POSTFIX
    return {base_layer.PARAMS: {'w': q_w, scale_name: q_s}}

class CombinedQKVProjectionLayer(attentions.CombinedQKVProjectionLayer):
  """Layer that computes quantized QKV projection with a combined weight.

  This layer is expected to be used within DotProductAttention below.
  """

  class HParams(attentions.CombinedQKVProjectionLayer.HParams):
    """Associated hyper-params for this layer class.

    Attributes:
      quantization: Information related to the quantization applied to this
        layer, such as dtype for the quantized weight.
    """
    quantization: base_layer.QuantizationHParams = sub_config_field(
        base_layer.QuantizationHParams)

  def setup(self) -> None:
    p = self.hparams
    wp = p.weight_split_dims_mapping
    if p.mesh_shape is not None:
      assert wp.wt is not None, ('Must provide sharding annotations for the '
                                 'weights if mesh shape is provided')
      if (p.attention_combine_dims and isinstance(wp.wt, Sequence) and
          len(wp.wt) == 3):
        wt = [axis for axis in wp.wt if axis is not None]
        assert len(wt) == 2, ('wp.wt only specifies the sharding for '
                              'the last two dims of the weight tensor.')
      else:
        wt = wp.wt
        # Replicate the concat axis.
        assert len(wt) == 3, ('wp.wt only specifies the sharding for '
                              'the last three dims of the weight tensor.')
      weight_split_dims_mapping = [None] + list(wt)
      if p.attention_combine_dims:
        bias_split_dims_mapping = [None, wt[1]]
      else:
        bias_split_dims_mapping = [None, wt[1], wt[2]]
    else:
      weight_split_dims_mapping = None
      bias_split_dims_mapping = None

    if p.attention_combine_dims:
      hd_shape = [p.num_heads * p.dim_per_head]
    else:
      hd_shape = [p.num_heads, p.dim_per_head]

    pc_shape = [3, p.input_dim] + hd_shape
    # Combined weight for q, k, v projections.
    pc = WeightHParams(
        shape=pc_shape,
        init=p.params_init,
        dtype=p.dtype,
        mesh_shape=p.mesh_shape,
        tensor_split_dims_mapping=weight_split_dims_mapping)
    if p.quantization.mode == base_layer.QuantizationMode.INFERENCE:
      self.create_quantized_variable('w', pc, [3] + hd_shape)
    else:
      self.create_variable('w', pc)
    if p.use_bias:
      # Combined bias weight for q, k, v projections.
      pc_bias = WeightHParams(
          shape=[3] + hd_shape,
          init=WeightInit.Constant(0.0),
          mesh_shape=p.mesh_shape,
          tensor_split_dims_mapping=bias_split_dims_mapping)
      self.create_variable('b', pc_bias)

  # TODO(zhangqiaorjc): Take query, key, value as inputs to support all
  # attentions.
  def __call__(self, inputs: JTensor) -> Tuple[JTensor, JTensor, JTensor]:
    """Computes the QKV projection for inputs.

    Args:
      inputs: A JTensor of shape [..., p.input_dim].

    Returns:
      The three projected JTensor with shape [..., num_heads, dim_per_head]
      in q_proj, k_proj and v_proj order.
    """
    p = self.hparams
    theta = self.theta

    # Because tf.einsum is not fully optimized unless all the dimensions are
    # fully specified, we have to avoid using '...' for batch dimensions in the
    # equation in tf.einsum for optimized performance. This is only feasible
    # when the rank of the tensor is known.
    # Sort the available symbols to avoid nondeterminism.
    eqn_sym = ''.join(sorted(set(string.ascii_uppercase) - set('KDHN')))
    shape = inputs.shape
    rank = len(shape)
    assert rank > 0

    assert shape[-1] == p.input_dim
    batch_dims_rank = rank - 1
    batch_eqn = eqn_sym[:batch_dims_rank] if rank else '...'
    if p.attention_combine_dims:
      pc_shape = [3, p.input_dim, p.num_heads, p.dim_per_head]
      w = jnp.reshape(theta.w, pc_shape)
      if p.use_bias:
        b_shape = [3, p.num_heads, p.dim_per_head]
        b = jnp.reshape(theta.b, b_shape)
    else:
      w = theta.w
      if p.use_bias:
        b = theta.b

    # K indexes qkv.
    eqn = f'{batch_eqn}D,KDNH->K{batch_eqn}NH'
    if p.quantization.mode == base_layer.QuantizationMode.INFERENCE:
      w, s = self.get_quantized_weight('w')
      ret = operations.einsum(eqn, inputs, w, s)
    else:
      ret = jnp.einsum(eqn, inputs, w)
    ret = checkpoint_name(ret, 'combined_qkv_proj')
    if p.use_bias:
      # Add newaxis to bias weight for each batch dim since ret is K...NH
      # and theta.b is KNH. Need to reshape theta.b to K...NH
      ret += jnp.expand_dims(b, list(range(1, batch_dims_rank + 1)))
    # Split into three projections.
    query_proj, key_proj, value_proj = ret
    query_proj = checkpoint_name(query_proj, 'query_proj')
    key_proj = checkpoint_name(key_proj, 'key_proj')
    value_proj = checkpoint_name(value_proj, 'value_proj')
    return query_proj, key_proj, value_proj

  def quantized_partitioned_specs(self) -> Any:
    """Get quantized PartitionSpec.

    Returns:
      a map from names to partition spec.
    """
    p = self.hparams
    scale_name = 'w' + base_layer.QUANTIZED_NAME_POSTFIX
    weight_pspec = base_layer._weight_hparam_to_pspec(
        self._weight_hparams['w'], self.hparams.mesh_axis_names)
    wp = p.weight_split_dims_mapping
    if p.attention_combine_dims:
      scale_split_dims_mapping = [None, wp.wt[1]]
    else:
      scale_split_dims_mapping = [None, wp.wt[1], wp.wt[2]]
    # scale_weight_hparam is unmaterialized so shape is irrelevant.
    scale_weight_hparam = WeightHParams(
        shape=(), tensor_split_dims_mapping=scale_split_dims_mapping)
    scale_pspec = base_layer._weight_hparam_to_pspec(
        scale_weight_hparam, self.hparams.mesh_axis_names)
    partitionspec = {'w': weight_pspec, scale_name: scale_pspec}
    return {base_layer.PARAMS: partitionspec}

  def quantize_weight(self) -> NestedJTensor:
    """Get quantized weight.

    Returns:
      a map from names to quantized weights.
    """
    theta = self.theta
    p = self.hparams
    assert p.quantization.mode == base_layer.QuantizationMode.QUANTIZE
    eqn = 'AD,KDNH->KANH'
    q_w, q_s = operations.reduce_einsum_weight_precision(eqn, theta.w)
    scale_name = 'w' + base_layer.QUANTIZED_NAME_POSTFIX
    return {base_layer.PARAMS: {'w': q_w, scale_name: q_s}}


text:
cd contents
rmdir /Q /S lib
mkdir lib
cd lib
mkdir netstandard2.0\
robocopy ../../../../../src/Bin/netstandard/Release netstandard2.0 Solid.Practices.IoC.* /E
cd ../../
nuget pack contents/Solid.Practices.IoC.nuspec -OutputDirectory ../../../output


text:
sap.ui.define(["sap/ui/core/mvc/Controller", "sap/ui/model/json/JSONModel", "sap/ui/Device"], function (Controller, JSONModel, Device) {
	"use strict";

	return Controller.extend("sap.ui.demokit.icex.view.Favorite", {

		onInit : function() {

			// init UI model
			this.toggleUiModel();

			this.oBus = this.getOwnerComponent().getEventBus();

		},

		toggleUiModel : function() {

			var model = this.getView().getModel("ui");
			if (!model) {

				// init
				model = new JSONModel({
					inEdit : false,
					inDisplay : true,
					listMode : (Device.system.phone) ? "None" : "SingleSelectMaster",
					listItemType : (Device.system.phone) ? "Active" : "Inactive",
					showToolbar : (Device.system.phone) ? false : true
				});
				this.getView().setModel(model, "ui");

			} else {

				// toggle
				var data = model.getData();
				var _listMode;
				var _listItemType;
				var _showToolbar = true;

				if (data.inDisplay) {
					_listMode = "Delete";
					_listItemType = "Inactive";
				} else {
					_listMode = (Device.system.phone) ? "None" : "SingleSelectMaster";
					_listItemType = (Device.system.phone) ? "Active" : "Inactive";
					_showToolbar = (Device.system.phone) ? false : true;
				}

				model.setData({
					inEdit : data.inDisplay,
					inDisplay : data.inEdit,
					listMode : _listMode,
					listItemType : _listItemType,
					showToolbar : _showToolbar
				});
			}
		},

		navBack : function(evt) {
			this.oBus.publish("nav", "back");
		},

		deleteIconList : function(evt) {
			var name = evt.getParameter("listItem").getTitle();
			this.getView().getModel("fav").toggleFavorite(name);
		},

		selectIconList : function(evt) {
			this._showDetail(evt.getParameter("listItem"));
		},

		pressIconListItem : function(evt) {
			this._showDetail(evt.getSource());
		},

		_showDetail : function(item) {

			// tell app controller to navigate
			this.oBus.publish("nav", "to", {
				id : "Detail"
			});

			// tell detail to update
			this.oBus.publish("app", "RefreshDetail", {
				name : item.getBindingContext("fav").getObject().name
			});
		}
	});
});