text:
package org.apache.carbondata.processing.store;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.ComplexColumnPage;
import org.apache.carbondata.core.datastore.page.EncodedTablePage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
import org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingFactory;
import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.EncodingFactory;
import org.apache.carbondata.core.datastore.page.statistics.KeyPageStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.LVLongStringStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.LVShortStringStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.datastore.row.ComplexColumnInfo;
import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
import org.apache.carbondata.core.localdictionary.generator.LocalDictionaryGenerator;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;

import org.apache.log4j.Logger;

/**
 * Represent a page data for all columns, we store its data in columnar layout, so that
 * all processing apply to TablePage can be done in vectorized fashion.
 */
public class TablePage {

  private static final Logger LOGGER =
      LogServiceFactory.getLogService(TablePage.class.getName());

  // For all dimension and measure columns, we store the column data directly in the page,
  // the length of the page is the number of rows.

  // TODO: we should have separate class for key columns so that keys are stored together in
  // one vector to make it efficient for sorting
  private ColumnPage[] dictDimensionPages;
  private ColumnPage[] noDictDimensionPages;
  private ColumnPage[] measurePages;
  private ComplexColumnPage[] complexDimensionPages;

  // the num of rows in this page, it must be less than short value (65536)
  private int pageSize;

  private CarbonFactDataHandlerModel model;

  private EncodedTablePage encodedTablePage;

  private EncodingFactory encodingFactory = DefaultEncodingFactory.getInstance();

  // true if it is last page of all input rows
  private boolean isLastPage;

  // used for complex column to deserilize the byte array in input CarbonRow
  private Map<Integer, GenericDataType> complexIndexMap = null;
  // name of compressor that used to compress column data,
  // currently all the columns share the same compressor.
  private String columnCompressor;

  TablePage(CarbonFactDataHandlerModel model, int pageSize) {
    this.model = model;
    this.pageSize = pageSize;
    TableSpec tableSpec = model.getTableSpec();
    this.columnCompressor = model.getColumnCompressor();

    dictDimensionPages = new ColumnPage[model.getDictDimensionCount()];
    noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()];
    int tmpNumDictDimIdx = 0;
    int tmpNumNoDictDimIdx = 0;
    for (int i = 0; i < tableSpec.getNumDimensions(); i++) {
      TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i);
      if (spec.getSchemaDataType().isComplexType()) {
        // skip complex columns and go other dimensions.
        // partition scenario dimensions can present after complex columns.
        continue;
      }
      ColumnPage page;
      if (spec.getSchemaDataType() == DataTypes.DATE) {
        page = ColumnPage.newPage(
            new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), pageSize);
        page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY));
        dictDimensionPages[tmpNumDictDimIdx++] = page;
      } else {
        // will be encoded using string page
        LocalDictionaryGenerator localDictionaryGenerator =
            model.getColumnLocalDictGenMap().get(spec.getFieldName());
        DataType dataType = DataTypes.STRING;
        if (DataTypes.VARCHAR == spec.getSchemaDataType()) {
          dataType = DataTypes.VARCHAR;
        } else if (DataTypes.BINARY == spec.getSchemaDataType()) {
          dataType = DataTypes.BINARY;
        }
        ColumnPageEncoderMeta columnPageEncoderMeta =
            new ColumnPageEncoderMeta(spec, dataType, columnCompressor);
        if (null != localDictionaryGenerator) {
          page = ColumnPage.newLocalDictPage(
              columnPageEncoderMeta, pageSize, localDictionaryGenerator, false);
        } else {
          if (DataTypeUtil.isPrimitiveColumn(spec.getSchemaDataType())) {
            if (spec.getSchemaDataType() == DataTypes.TIMESTAMP) {
              columnPageEncoderMeta =
                  new ColumnPageEncoderMeta(spec, DataTypes.LONG, columnCompressor);
            } else {
              columnPageEncoderMeta =
                  new ColumnPageEncoderMeta(spec, spec.getSchemaDataType(), columnCompressor);
            }
            // create the column page according to the data type for no dictionary numeric columns
            if (DataTypes.isDecimal(spec.getSchemaDataType())) {
              page = ColumnPage.newDecimalPage(columnPageEncoderMeta, pageSize);
            } else {
              page = ColumnPage.newPage(columnPageEncoderMeta, pageSize);
            }
          } else {
            page = ColumnPage.newPage(columnPageEncoderMeta, pageSize);
          }
        }
        // set the stats collector according to the data type of the columns
        if (DataTypes.VARCHAR == dataType || DataTypes.BINARY == dataType) {
          page.setStatsCollector(LVLongStringStatsCollector.newInstance());
        } else if (DataTypeUtil.isPrimitiveColumn(spec.getSchemaDataType())) {
          if (spec.getSchemaDataType() == DataTypes.TIMESTAMP) {
            page.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.LONG));
          } else {
            page.setStatsCollector(
                PrimitivePageStatsCollector.newInstance(spec.getSchemaDataType()));
          }
        } else {
          page.setStatsCollector(LVShortStringStatsCollector.newInstance());
        }
        noDictDimensionPages[tmpNumNoDictDimIdx++] = page;
      }
    }
    complexDimensionPages = new ComplexColumnPage[model.getComplexColumnCount()];
    for (int i = 0; i < complexDimensionPages.length; i++) {
      // here we still do not the depth of the complex column, it will be initialized when
      // we get the first row.
      complexDimensionPages[i] = null;
    }
    measurePages = new ColumnPage[model.getMeasureCount()];
    DataType[] dataTypes = model.getMeasureDataType();
    for (int i = 0; i < measurePages.length; i++) {
      ColumnPageEncoderMeta columnPageEncoderMeta = new ColumnPageEncoderMeta(
          model.getTableSpec().getMeasureSpec(i), dataTypes[i], columnCompressor);
      ColumnPage page;
      if (DataTypes.isDecimal(columnPageEncoderMeta.getSchemaDataType())) {
        page = ColumnPage.newDecimalPage(columnPageEncoderMeta, pageSize);
      } else {
        page = ColumnPage.newPage(columnPageEncoderMeta, pageSize);
      }
      page.setStatsCollector(PrimitivePageStatsCollector.newInstance(dataTypes[i]));
      measurePages[i] = page;
    }

    // for complex type, `complexIndexMap` is used in multithread (in multiple Producer),
    // we need to clone the index map to make it thread safe
    this.complexIndexMap = new HashMap<>();
    for (Map.Entry<Integer, GenericDataType> entry: model.getComplexIndexMap().entrySet()) {
      this.complexIndexMap.put(entry.getKey(), entry.getValue().deepCopy());
    }
  }

  /**
   * Add one row to the internal store
   *
   * @param rowId Id of the input row
   * @param row   row object
   */
  public void addRow(int rowId, CarbonRow row) {
    // convert each column category, update key and stats
    convertToColumnarAndAddToPages(rowId, row);
  }

  // convert the input row object to columnar data and add to column pages
  private void convertToColumnarAndAddToPages(int rowId, CarbonRow row) {
    // 1. convert dictionary columns
    int[] dictDimensions = WriteStepRowUtil.getDictDimension(row);
    byte[][] keys = new byte[dictDimensions.length][];
    for (int i = 0; i < dictDimensions.length; i++) {
      keys[i] = ByteUtil.toBytes(dictDimensions[i]);
    }
    for (int i = 0; i < dictDimensionPages.length; i++) {
      dictDimensionPages[i].putData(rowId, keys[i]);
    }

    // 2. convert noDictionary columns and complex columns and varchar, binary columns.
    int noDictionaryCount = noDictDimensionPages.length;
    int complexColumnCount = complexDimensionPages.length;
    if (noDictionaryCount > 0 || complexColumnCount > 0) {
      TableSpec tableSpec = model.getTableSpec();
      List<TableSpec.DimensionSpec> noDictionaryDimensionSpec =
          tableSpec.getNoDictionaryDimensionSpec();
      Object[] noDictAndComplex = WriteStepRowUtil.getNoDictAndComplexDimension(row);
      for (int i = 0; i < noDictAndComplex.length; i++) {
        if (noDictionaryDimensionSpec.get(i).getSchemaDataType() == DataTypes.VARCHAR
            || noDictionaryDimensionSpec.get(i).getSchemaDataType() == DataTypes.BINARY) {
          byte[] valueWithLength = addIntLengthToByteArray((byte[]) noDictAndComplex[i]);
          noDictDimensionPages[i].putData(rowId, valueWithLength);
        } else if (i < noDictionaryCount) {
          if (DataTypeUtil
              .isPrimitiveColumn(noDictDimensionPages[i].getColumnSpec().getSchemaDataType())) {
            // put the actual data to the row
            Object value = noDictAndComplex[i];
            // in compaction flow the measure with decimal type will come as Spark decimal.
            // need to convert it to byte array.
            if (DataTypes.isDecimal(noDictDimensionPages[i].getDataType()) && model
                .isCompactionFlow() && value != null) {
              value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);
            }
            noDictDimensionPages[i].putData(rowId, value);
          } else {
            // noDictionary columns, since it is variable length, we need to prepare each
            // element as LV result byte array (first two bytes are the length of the array)
            byte[] valueWithLength = addShortLengthToByteArray((byte[]) noDictAndComplex[i]);
            noDictDimensionPages[i].putData(rowId, valueWithLength);
          }
        } else {
          // complex columns
          addComplexColumn(i - noDictionaryCount, rowId,
              (List<ArrayList<byte[]>>) noDictAndComplex[i]);
        }
      }
    }
    // 3. convert measure columns
    Object[] measureColumns = WriteStepRowUtil.getMeasure(row);
    for (int i = 0; i < measurePages.length; i++) {
      Object value = measureColumns[i];

      // in compaction flow the measure with decimal type will come as Spark decimal.
      // need to convert it to byte array.
      if (DataTypes.isDecimal(measurePages[i].getDataType()) &&
          model.isCompactionFlow() &&
          value != null) {
        value = DataTypeUtil.getDataTypeConverter().convertFromDecimalToBigDecimal(value);
      }
      measurePages[i].putData(rowId, value);
    }
  }

  /**
   * add a complex column into internal member complexDimensionPage
   *
   * @param index          index of the complexDimensionPage
   * @param rowId          Id of the input row
   * @param encodedComplexColumnar flatten data of complex column
   */
  // TODO: this function should be refactoried, ColumnPage should support complex type encoding
  // directly instead of doing it here
  private void addComplexColumn(int index, int rowId,
      List<ArrayList<byte[]>> encodedComplexColumnar) {
    GenericDataType complexDataType = complexIndexMap.get(
        index + model.getSegmentProperties().getNumberOfPrimitiveDimensions());
    // initialize the page if first row
    if (rowId == 0) {
      List<ComplexColumnInfo> complexColumnInfoList = new ArrayList<>();
      complexDataType.getComplexColumnInfo(complexColumnInfoList);
      complexDimensionPages[index] = new ComplexColumnPage(complexColumnInfoList);
      complexDimensionPages[index].initialize(
          model.getColumnLocalDictGenMap(), pageSize, columnCompressor);
    }
    int depthInComplexColumn = complexDimensionPages[index].getComplexColumnIndex();
    for (int depth = 0; depth < depthInComplexColumn; depth++) {
      complexDimensionPages[index].putComplexData(depth, encodedComplexColumnar.get(depth));
    }
  }

  void freeMemory() {
    for (ColumnPage page : dictDimensionPages) {
      page.freeMemory();
    }
    for (ColumnPage page : noDictDimensionPages) {
      page.freeMemory();
    }
    for (ColumnPage page : measurePages) {
      page.freeMemory();
    }
    for (ComplexColumnPage page : complexDimensionPages) {
      if (null != page) {
        page.freeMemory();
      }
    }
  }

  // Adds length as a short element (first 2 bytes) to the head of the input byte array
  private byte[] addShortLengthToByteArray(byte[] input) {
    if (input.length > Short.MAX_VALUE) {
      throw new RuntimeException("input data length " + input.length +
          " bytes too long, maximum length supported is " + Short.MAX_VALUE + " bytes");
    }
    byte[] output = new byte[input.length + 2];
    ByteBuffer buffer = ByteBuffer.wrap(output);
    buffer.putShort((short)input.length);
    buffer.put(input, 0, input.length);
    return output;
  }

  // Adds length as a integer element (first 4 bytes) to the head of the input byte array
  private byte[] addIntLengthToByteArray(byte[] input) {
    byte[] output = new byte[input.length + 4];
    ByteBuffer buffer = ByteBuffer.wrap(output);
    buffer.putInt(input.length);
    buffer.put(input, 0, input.length);
    return output;
  }

  void encode() throws IOException {
    // encode dimensions and measure
    EncodedColumnPage[] dimensions = encodeAndCompressDimensions();
    EncodedColumnPage[] measures = encodeAndCompressMeasures();
    this.encodedTablePage = EncodedTablePage.newInstance(pageSize, dimensions, measures);
  }

  public EncodedTablePage getEncodedTablePage() {
    return encodedTablePage;
  }

  // apply measure and set encodedData in `encodedData`
  private EncodedColumnPage[] encodeAndCompressMeasures() throws IOException {
    EncodedColumnPage[] encodedMeasures = new EncodedColumnPage[measurePages.length];
    for (int i = 0; i < measurePages.length; i++) {
      ColumnPageEncoder encoder = encodingFactory.createEncoder(
          model.getTableSpec().getMeasureSpec(i), measurePages[i]);
      encodedMeasures[i] = encoder.encode(measurePages[i]);
    }
    return encodedMeasures;
  }

  // apply and compress each dimension, set encoded data in `encodedData`
  private EncodedColumnPage[] encodeAndCompressDimensions() throws IOException {
    List<EncodedColumnPage> encodedDimensions = new ArrayList<>();
    List<EncodedColumnPage> encodedComplexDimensions = new ArrayList<>();
    TableSpec tableSpec = model.getTableSpec();
    int dictIndex = 0;
    int noDictIndex = 0;
    int complexDimIndex = 0;
    int numDimensions = tableSpec.getNumDimensions();
    for (int i = 0; i < numDimensions; i++) {
      ColumnPageEncoder columnPageEncoder;
      EncodedColumnPage encodedPage;
      TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i);
      switch (spec.getColumnType()) {
        case DIRECT_DICTIONARY:
          columnPageEncoder = encodingFactory.createEncoder(
              spec,
              dictDimensionPages[dictIndex]);
          encodedPage = columnPageEncoder.encode(dictDimensionPages[dictIndex++]);
          encodedDimensions.add(encodedPage);
          break;
        case PLAIN_VALUE:
          columnPageEncoder = encodingFactory.createEncoder(
              spec,
              noDictDimensionPages[noDictIndex]);
          encodedPage = columnPageEncoder.encode(noDictDimensionPages[noDictIndex]);
          if (LOGGER.isDebugEnabled()) {
            DataType targetDataType =
                columnPageEncoder.getTargetDataType(noDictDimensionPages[noDictIndex]);
            if (null != targetDataType) {
              LOGGER.debug(
                  "Encoder result ---> Source data type: " + noDictDimensionPages[noDictIndex]
                      .getDataType().getName() + " Destination data type: " + targetDataType
                      .getName() + " for the column: " + noDictDimensionPages[noDictIndex]
                      .getColumnSpec().getFieldName() + " having encoding type: "
                      + columnPageEncoder.getEncodingType());
            }
          }
          noDictIndex++;
          encodedDimensions.add(encodedPage);
          break;
        case COMPLEX:
          EncodedColumnPage[] encodedPages = ColumnPageEncoder.encodeComplexColumn(
              complexDimensionPages[complexDimIndex++]);
          encodedComplexDimensions.addAll(Arrays.asList(encodedPages));
          break;
        default:
          throw new IllegalArgumentException("unsupported dimension type:" + spec
              .getColumnType());
      }
    }

    encodedDimensions.addAll(encodedComplexDimensions);
    return encodedDimensions.toArray(new EncodedColumnPage[encodedDimensions.size()]);
  }

  /**
   * return column page of specified column name
   */
  public ColumnPage getColumnPage(String columnName) {
    int dictDimensionIndex = -1;
    int noDictDimensionIndex = -1;
    ColumnPage page = null;
    TableSpec spec = model.getTableSpec();
    int numDimensions = spec.getNumDimensions();
    for (int i = 0; i < numDimensions; i++) {
      ColumnType type = spec.getDimensionSpec(i).getColumnType();
      if (spec.getDimensionSpec(i).getSchemaDataType() == DataTypes.DATE) {
        page = dictDimensionPages[++dictDimensionIndex];
      } else if (type == ColumnType.PLAIN_VALUE) {
        page = noDictDimensionPages[++noDictDimensionIndex];
      } else {
        // do not support datamap on complex column
        continue;
      }
      String fieldName = spec.getDimensionSpec(i).getFieldName();
      if (fieldName.equalsIgnoreCase(columnName)) {
        return page;
      }
    }
    int numMeasures = spec.getNumMeasures();
    for (int i = 0; i < numMeasures; i++) {
      String fieldName = spec.getMeasureSpec(i).getFieldName();
      if (fieldName.equalsIgnoreCase(columnName)) {
        return measurePages[i];
      }
    }
    throw new IllegalArgumentException("DataMap: must have '" + columnName + "' column in schema");
  }

  public boolean isLastPage() {
    return isLastPage;
  }

  public void setIsLastPage(boolean isWriteAll) {
    this.isLastPage = isWriteAll;
  }

  public int getPageSize() {
    return pageSize;
  }
}


text: require 'wordify_aatkinson'

redpajama_set_name: RedPajamaGithub


text:
""" Name that Shape module for use with Lab 5, Inf1340, Fall 2015 """

__author__ = 'Susan Sim'
__email__ = "ses@drsusansim.org"
__copyright__ = "2015 Susan Sim"
__license__ = "MIT License"

def name_that_shape(sides):
    """
    For a given number of sides in a regular polygon, returns the shape name

    Inputs | Expected Outputs
    -------------------------
      < 3  | Error
      3    | triangle
      4    | quadrilateral
      5    | pentagon
      6    | hexagon
      7    | heptagon
      8    | octagon
      9    | nonagon
      10   | decagon
      > 10 | Error

    Errors: TypeError when input is a string or float
            ValueError when input is < 3 or > 10

    """
    # Some Changes made
    sides = str(sides)
    if sides.isdigit() or sides[0] is "-" and sides[1:].isdigit():
        sides = int(sides)
    else:
        return None

    if sides == 3:
        return("triangle")
    elif sides == 4:
        return("quadrilateral")
    elif sides == 5:
        return("pentagon")
    elif sides == 6:
        return("hexagon")
    elif sides == 7:
        return("heptagon")
    elif sides == 8:
        return("octagon")
    elif sides == 9:
        return("nonagon")
    elif sides == 10:
        return("decagon")
    else:
        return None


text:
"""Demonstrates how to approve a Merchant Center link request.

Prerequisite: You need to have access to a Merchant Center account. You can find
instructions to create a Merchant Center account here:
https://support.google.com/merchants/answer/188924.

To run this code example, you must use the Merchant Center UI or the Content
API for Shopping to send a link request between your Merchant Center and Google
Ads accounts.
"""

import argparse
import sys

from google.ads.googleads.client import GoogleAdsClient
from google.ads.googleads.errors import GoogleAdsException
from google.api_core import protobuf_helpers

def main(client, customer_id, merchant_center_account_id):
    """Demonstrates how to approve a Merchant Center link request.

    Args:
        client: An initialized GoogleAdsClient instance.
        customer_id: The client customer ID string.
        merchant_center_account_id: ID of the Merchant center whose link request
        is to be approved.
    """
    merchant_center_link_service = client.get_service(
        "MerchantCenterLinkService"
    )

    # [START approve_merchant_center_link]
    # Retrieve all the existing Merchant Center links.
    response = merchant_center_link_service.list_merchant_center_links(
        customer_id=customer_id
    )
    print(
        f"{len(response.merchant_center_links)} Merchant Center link(s) "
        "found with the following details:"
    )
    # [END approve_merchant_center_link]

    merchant_center_link_status_enum = client.enums.MerchantCenterLinkStatusEnum

    # Iterate through the results and filter for links with pending statuses.
    for merchant_center_link in response.merchant_center_links:
        # [START approve_merchant_center_link_1]
        print(
            f"Link '{merchant_center_link.resource_name}' has status "
            f"'{merchant_center_link.status.name}'."
        )
        # [END approve_merchant_center_link_1]

        if (
            merchant_center_link.status
            == merchant_center_link_status_enum.PENDING
            and str(merchant_center_link.id) == merchant_center_account_id
        ):
            update_merchant_center_link_status(
                client,
                customer_id,
                merchant_center_link_service,
                merchant_center_link,
                merchant_center_link_status_enum.ENABLED,
            )

# [START approve_merchant_center_link_2]
def update_merchant_center_link_status(
    client,
    customer_id,
    merchant_center_link_service,
    merchant_center_link,
    status,
):
    """Updates the status of a Merchant Center link request.

    Args:
        client: An initialized GoogleAdsClient instance.
        customer_id: The client customer ID string.
        merchant_center_link_service: A merchant center link service instance.
        merchant_center_link: The merchant center link to be modified.
        status: The updated status to apply to the merchant center link.
    """
    # Creates an operation.
    operation = client.get_type("MerchantCenterLinkOperation")
    link_to_update = operation.update
    link_to_update.resource_name = merchant_center_link.resource_name
    # Enables the pending link.
    link_to_update.status = status
    client.copy_from(
        operation.update_mask,
        protobuf_helpers.field_mask(None, link_to_update._pb),
    )

    # Updates the link.
    mutate_response = merchant_center_link_service.mutate_merchant_center_link(
        customer_id=customer_id, operation=operation
    )

    # Displays the result.
    print(
        "The status of Merchant Center Link with resource name "
        f"'{mutate_response.result.resource_name}' to Google Ads account : "
        f"{customer_id} was updated to {status.name}."
    )
    # [END approve_merchant_center_link_2]

if __name__ == "__main__":
    # GoogleAdsClient will read the google-ads.yaml configuration file in the
    # home directory if none is specified.
    googleads_client = GoogleAdsClient.load_from_storage(version="v12")

    parser = argparse.ArgumentParser(
        description=("Approves a Merchant Center link request.")
    )
    # The following argument(s) should be provided to run the example.
    parser.add_argument(
        "-c",
        "--customer_id",
        type=str,
        required=True,
        help="The Google Ads customer ID.",
    )
    parser.add_argument(
        "-m",
        "--merchant_center_account_id",
        type=str,
        required=True,
        help="ID of the Merchant Center account whose link request is to be "
        "approved.",
    )
    args = parser.parse_args()

    try:
        main(
            googleads_client, args.customer_id, args.merchant_center_account_id
        )
    except GoogleAdsException as ex:
        print(
            f'Request with ID "{ex.request_id}" failed with status'
            f'"{ex.error.code().name}" and includes the following errors:'
        )
        for error in ex.failure.errors:
            print(f'\tError with message "{error.message}".')
            if error.location:
                for field_path_element in error.location.field_path_elements:
                    print(f"\t\tOn field: {field_path_element.field_name}")
        sys.exit(1)


text:
from __future__ import unicode_literals

from datetime import datetime
from ..models import GitHubCore
from ..users import User

def alternate_week(week):
    return {
        'start of week': datetime.fromtimestamp(int(week['w'])),
        'additions': week['a'],
        'deletions': week['d'],
        'commits': week['c'],
    }

class ContributorStats(GitHubCore):

    """This object provides easy access to information returned by the
    statistics section of the API.

    See http://developer.github.com/v3/repos/statistics/ for specifics.

    """

    def __init__(self, stats_object, session=None):
        super(ContributorStats, self).__init__(stats_object, session)
        #: Contributor in particular that this relates to
        self.author = User(stats_object.get('author', {}), session)
        #: Total number of commits authored by ``author``.
        self.total = stats_object.get('total')
        #: List of weekly dictionaries.
        self.weeks = stats_object.get('weeks', [])
        #: Alternative collection of weekly dictionaries
        #: This provides a datetime object and easy to remember keys for each
        #: element in the list. 'w' -> 'start of week', 'a' -> 'Number of additions',
        #: 'd' -> 'Number of deletions', 'c' -> 'Number of commits'
        self.alt_weeks = [alternate_week(w) for w in self.weeks]

    def _repr(self):
        return '<Contributor Statistics [{0}]>'.format(self.author)


text:
import os, pathlib, subprocess, time

CURRENT_DIRECTORY = 0
COMPILE_MESSAGE_LINE_LIMIT = 10

def in_each_repo(function, root_path = CURRENT_DIRECTORY):
    if root_path == CURRENT_DIRECTORY:
        root_path = pathlib.PosixPath()
    for child in root_path.iterdir():
        # skip __pycache__
        if child.is_dir() and child != '__pycache__':
            in_subdir(lambda path: assess_one_repo(function), child)
    print_banner()

def in_subdir(function, child_name):
    current_dir = current_path()
    subdir = current_dir / child_name
    try:
        os.chdir(subdir)
        result = function(subdir)
        os.chdir(current_dir)
        return result
    except FileNotFoundError:
        print('ERROR no such directory "' + child_name + '"')
        return None

def assess_one_repo(function):
    print_banner()
    parent = current_path()
    repo_dir_name = parent.parts[-1]
    print(str(repo_dir_name))
    return function()

def current_path():
    return pathlib.PosixPath(os.getcwd())

def print_banner():
    print('=' * 79)

def print_repo_name(path):
    print(path.parts[-1])

# return True on success, False on failure
def print_line_number(path, line_number_from_1):
    ok = True
    print(' line ' + str(line_number_from_1) + ' of ' + str(path) + ':')
    try:
        with open(path) as f:
            lines = f.readlines()
            index = line_number_from_1 - 1
            if (index < 0) or (index >= len(lines)):
                print('(that line does not exist)')
                ok = False
            else:
                print(lines[index])
    except OSError:
        print('(file does not exist)')
        ok = False
    return ok

# return True if there was at least one match, False otherwise
def print_matching_lines(path, substring):
    found_match = False
    try:
        with open(path) as f:
            lines = f.readlines()
            for i in range(len(lines)):
                line = lines[i]
                if substring in line:
                    print('line ' + str(i+1) + ' of ' + str(path) + ': ' + line)
                    found_match = True
    except OSError:
        print('(' + str(path) + ' does not exist)')
    return found_match

def print_file(path):
    try:
        with open(path) as f:
            print(f.read())
    except OSError:
        print('(' + str(path) + ' does not exist)')

def compiles_cleanly_in_subdir(subdir, source_filename):
    in_subdir(lambda path: compiles_cleanly(source_filename), subdir)

def compiles_cleanly(source_filename):
    result = True
    print(str(source_filename) + ' compiles cleanly:', end='')
    messages = command(['g++', source_filename])
    if not messages:
        print('YES')
    else:
        print('NO, messages:')
        for line in messages.splitlines()[:COMPILE_MESSAGE_LINE_LIMIT]:
            print(line)
        result = False
    print('')
    return result

# return a tuple (output, exit_code)
def output_and_exit_code(args, stdin_string=None, timeout=None):
    try:
        proc = subprocess.run(args,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT,
                              text=True,
                              input=stdin_string,
                              timeout=timeout)
        return (proc.stdout, proc.returncode)
    except subprocess.TimeoutExpired:
        print("TIMEOUT AFTER " + str(timeout) + " seconds")
        return ('', 255)

# If the command succeeds (exit code 0), return None.
# Otherwise return a string with stdout and stderr, probably containing error messages.
def command(args):
    messages, exit_code = output_and_exit_code(args)
    if exit_code == 0:
        return None
    else:
        return messages

def print_output_and_exit_code_in_subdir(subdir, source_filename):
    print('running ' + source_filename + '...')

    def subdir_stuff(path):
        print('  ', end='')
        if compiles_cleanly(source_filename):
            output, exit_code = output_and_exit_code(['./a.out'])
            print('  EXIT CODE = ' + str(exit_code), 'OUTPUT ON NEXT LINE:')
            print(output + '(end of output)\n')

    in_subdir(subdir_stuff, subdir)

def commandline_test_case_in_subdir(subdir, source_filename, arguments, stdin_string=None, timeout=None):
    print('running ' + source_filename + '...')

    def subdir_stuff(path):
        print('  ', end='')
        if compiles_cleanly(source_filename):
            command = ['./a.out'] + arguments
            print('$ ./a.out ' + ' '.join(arguments))
            output, exit_code = output_and_exit_code(command, stdin_string, timeout)
            print('-OUTPUT BEGINS-')
            print(output)
            print('-OUTPUT ENDS-')

    in_subdir(subdir_stuff, subdir)

def git_log():
    output, exit_code = output_and_exit_code(['git', 'log'])
    print('git log:\n' + output)

# default seconds is slightly larger than 1 so that C++ srand(time(0)) will
# have a different seed.
def repeat_with_delay(function, count, seconds=1.01):
    for i in range(count):
        if i > 0:
            time.sleep(seconds)
        function()


text:
require 'spec_helper'

describe OrganizationMembershipsController do
  let(:user) { FactoryBot.create(:super_admin_user) }
  before do
    @config = FactoryBot.create(:event_configuration, :with_usa)
    sign_in user

    FactoryBot.create_list(:competitor, 5)
  end

  let(:registrant) { FactoryBot.create(:registrant) }
  let(:organization_membership) { registrant.organization_membership }

  describe "#index" do
    it "can list all members" do
      get :index
      expect(response).to be_successful
    end
  end

  describe "#toggle_confirm" do
    context "when the user is not yet confirmed" do
      it "can confirm" do
        put :toggle_confirm, params: { id: registrant.id }
        expect(registrant.reload.organization_membership).to be_manually_confirmed
      end
      it "can confirm with JS" do
        put :toggle_confirm, params: { id: registrant.id, format: :js }
        expect(registrant.reload.organization_membership).to be_manually_confirmed
      end
    end

    context "when the user is a confirmed member" do
      before { organization_membership.update_attribute(:manually_confirmed, true) }

      it "can unconfirm" do
        put :toggle_confirm, params: { id: registrant.id }
        expect(registrant.reload.organization_membership).not_to be_manually_confirmed
      end
    end
  end

  describe "#update_number" do
    let(:new_member_number) { "10" }

    it "saves the membership number" do
      put :update_number, params: { id: registrant.id, membership_number: new_member_number }, format: :js
      expect(registrant.reload.organization_membership.manual_member_number).to eq new_member_number
    end
  end

  describe "#export" do
    it "outputs some rows" do
      get :export, format: 'xls'
      expect(response.content_type.to_s).to eq("application/vnd.ms-excel")
    end
  end

  describe "#refresh_organization_status" do
    it "returns success" do
      allow_any_instance_of(MembershipChecker::Usa).to receive(:current_member?).and_return(true)
      post :refresh_organization_status, params: { id: registrant.id, format: :js }
      expect(response).to be_successful
    end
  end
end


text: This app demonstrates bookmarking with the state encoded in the URL.


text:
using System.Collections.Generic;
using Fpm.ProfileData;

namespace Fpm.MainUI.Models
{
    public class RemoveIndicatorsModel : BaseDataModel
    {
        // Filtering-related properties
        public string ProfileName { get; set; }
        public string DomainName { get; set; }
        public int DomainId { get; set; }
        public int SelectedAreaTypeId { get; set; }

        public IEnumerable<GroupingPlusName> IndicatorsThatCantBeRemoved { get; set; }
        public IList<GroupingPlusName> IndicatorsToRemove { get; set; }
    }
}


text:
package com.compositesw.services.system.admin.resource;

import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlType;

/**
 *
Java class for lineageResourceList complex type.
 *
 *
The following schema fragment specifies the expected content contained within this class.
 *
 *

 * <complexType name="lineageResourceList">
 *   <complexContent>
 *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
 *       <sequence>
 *         <element name="lineageResource" type="{http://www.compositesw.com/services/system/admin/resource}lineageResource" maxOccurs="unbounded" minOccurs="0"/>
 *       </sequence>
 *     </restriction>
 *   </complexContent>
 * </complexType>
 *

 *
 *
 */
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "lineageResourceList", propOrder = {
    "lineageResource"
})
public class LineageResourceList {

    protected List<LineageResource> lineageResource;

    /**
     * Gets the value of the lineageResource property.
     *
     *

     * This accessor method returns a reference to the live list,
     * not a snapshot. Therefore any modification you make to the
     * returned list will be present inside the JAXB object.
     * This is why there is not a set method for the lineageResource property.
     *
     *

     * For example, to add a new item, do as follows:
     *

     *    getLineageResource().add(newItem);
     *

     *
     *
     *

     * Objects of the following type(s) are allowed in the list
     * {@link LineageResource }
     *
     *
     */
    public List<LineageResource> getLineageResource() {
        if (lineageResource == null) {
            lineageResource = new ArrayList<LineageResource>();
        }
        return this.lineageResource;
    }

}


text:
<!DOCTYPE frameset SYSTEM "frameset.dtd">
<frameset>

    <predicate lemma="desaturation">

        <roleset framnet="" id="desaturation.101" name="Decreasing oxygen saturation in hemoglobin" vncls="">
            <roles>
                <role descr="Agent" f="" n="0"/>

                <role descr="Entity undergoing desaturation" f="" n="1"/>
                <role descr="Instrument" f="" n="2"/>
                <note/>
            </roles>

            <note/>
            <example name="Arg1" src="" type="">
                <inflection aspect="ns" form="ns" person="ns" tense="ns" voice="ns"/>
                <text>A very low-calorie diet can result in desaturation of skeletal muscle.</text>
                <note/>
                <rel f="">desaturation</rel>
                <arg f="" n="1">of skeletal muscle</arg>
            </example>
        </roleset>
        <note/>
    </predicate>
    <note/>
</frameset>


text: first stellajs game (ship shooter)


text:
package p1;

/**
 * This is the class documentation.
 * Testing multiple @since and @version tags only to see if all
 * instances of SimpleTaglet have the same bug
 *
 * @author Doug Kramer
 * @author Jamie
 * @author Neal
 *
 * @since  JDK 1.0
 *
 * @version 1.1
 * @version 1.2
 * @version 1.3
 */
public class C1 {
}


text:
const program = require('commander');
const build = require('./index');
const pkg = require('./package');

program
  .version(pkg.version)
  .description(pkg.description)
  .usage('[options] <entrypoint>')
  .option('-o, --out <file>', 'output file (default to dev-bundle.js)', 'dev-bundle.js')
  .parse(process.argv);

const expression = program.args[0];
if (!expression) program.help();

build({
  expression: expression,
  outLoc: program.out,
});


text:
package com.binarywang.spring.starter.wxjava.open.config;

import me.chanjar.weixin.open.api.WxOpenComponentService;
import me.chanjar.weixin.open.api.WxOpenConfigStorage;
import me.chanjar.weixin.open.api.WxOpenService;
import me.chanjar.weixin.open.api.impl.WxOpenMessageRouter;
import me.chanjar.weixin.open.api.impl.WxOpenServiceImpl;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * 微信开放平台相关服务自动注册.
 *
 * @author someone
 */
@Configuration
public class WxOpenServiceAutoConfiguration {

  @Bean
  @ConditionalOnMissingBean
  @ConditionalOnBean(WxOpenConfigStorage.class)
  public WxOpenService wxOpenService(WxOpenConfigStorage wxOpenConfigStorage) {
    WxOpenService wxOpenService = new WxOpenServiceImpl();
    wxOpenService.setWxOpenConfigStorage(wxOpenConfigStorage);
    return wxOpenService;
  }

  @Bean
  public WxOpenMessageRouter wxOpenMessageRouter(WxOpenService wxOpenService) {
    return new WxOpenMessageRouter(wxOpenService);
  }

  @Bean
  public WxOpenComponentService wxOpenComponentService(WxOpenService wxOpenService) {
    return wxOpenService.getWxOpenComponentService();
  }
}


text:
require 'shoulda-matchers'

module Shoulda
  module Matchers
    module ActiveModel
      def require_a_valid_domain_name(attribute = :domain_name)
        RequireAValidDomainNameMatcher.new(attribute)
      end

      class RequireAValidDomainNameMatcher < ValidationMatcher
        def description
          "require #{@attribute} to be a valid domain name"
        end

        def matches?(subject)
          super(subject)

          disallows_invalid_value && allows_valid_value
        end

        private

        def disallows_invalid_value
          disallows_value_of('example')
        end

        def allows_valid_value
          allows_value_of('example.com')
        end
      end
    end
  end
end


text:
angular.module('ifocus.jump')
  .factory('FlowJumpService', ['$http', 'baseUrl', function ($http, baseUrl) {

    return {
      fetchJumpingDataByStage : function (stage) {
        return $http.get(baseUrl + 'visitflow/path/jumpInfo/' + stage);
      }

    };

  }]);


text:
<?php

namespace Estoque\Entity;

use Doctrine\ORM\Mapping as ORM;
use Zend\InputFilter\InputFilterAwareInterface;
use Zend\InputFilter\InputFilterInterface;
use Zend\InputFilter\InputFilter;

/**
* @ORM\Entity(repositoryClass="\Estoque\Entity\Repository\ProdutoRepository")
* @ORM\Table(name="produto")
*/
class Produto implements InputFilterAwareInterface{

	/**
	* @ORM\Id
	* @ORM\Column(type="integer",nullable=true)
	* @ORM\GeneratedValue(strategy="AUTO")
	*/
	private $id;

	/**
	* @ORM\Column(type="string",nullable=true)
	*/
	private $nome;

	/**
	* @ORM\Column(type="decimal",scale=2,nullable=true)
	*/
	private $preco;

	/**
	* @ORM\Column(type="text",nullable=true)
	*/
	private $descricao;

    /**
    * @ORM\ManyToOne(targetEntity="Estoque\Entity\Categoria", inversedBy="produto")
    * @ORM\JoinColumn(name="categoria_id",referencedColumnName="id", nullable=false)
    */
    private $categoria;

	/**
	 * @ORM\Column(type="integer",nullable=true)
	 */
	private $registro = 1;

	/**
	 * @ORM\Column(type="datetime",nullable=true)
	 */
	private $dhinclusao;

	public function __construct($nome,$preco,$descricao){
		$this->nome = $nome;
		$this->preco = $preco;
		$this->descricao = $descricao;
	}

	public function getId() {
		return $this->id;
	}

	public function getNome() {
		return $this->nome;
	}

	public function getPreco() {
		return $this->preco;
	}

	public function getDescricao() {
		return $this->descricao;
	}

    public function setCategoria(Categoria $categoria)
    {
        $this->categoria = $categoria;
    }

    public function getCategoria()
    {
        return $this->categoria;
    }

	public function getRegistro() {
		return $this->registro;
	}

	public function getDhInclusao() {
		return $this->dhinclusao;
	}

	public function setNome($nome){
		$this->nome = $nome;
	}

	public function setPreco($preco){
		$this->preco = $preco;
	}

	public function setDescricao($descricao){
		$this->descricao = $descricao;
	}

	public function setRegistro($valor){
		$this->registro = (int) $valor;
	}

    public function setInputFilter(InputFilterInterface $inputFilter)
    {
        throw new Exception("Você não deve invocar este método");
    }

    public function getInputFilter()
    {
        $inputFilter = new InputFilter();

        $inputFilter->add([
            'name' => 'nome',
            'required' => true,
            'validators' => [
                [
                    'name' => 'not_empty'
                ],
                [
                    'name' => 'StringLength',
                    'options' => [
                        'min' => 3,
                        'max' => 100
                    ]
                ]
            ]
        ]);

        $inputFilter->add([
            'name' => 'preco',
            'required' => true,
            'validators' => [
                [
                    'name' => 'not_empty'
                ],
                [
                    'name' => 'IsFloat',
                ]
            ]
        ]);

        $inputFilter->add([
            'name' => 'categoria',
            'required' => true,
            'validators' => [
                [
                    'name' => 'not_empty'
                ],
            ]
        ]);

        return $inputFilter;
    }
}

?>


text:
require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')

describe AccountContact do
  before(:each) do
    @valid_attributes = {
      :account => mock_model(Account),
      :contact => mock_model(Contact)
    }
  end

  it "should create a new instance given valid attributes" do
    AccountContact.create!(@valid_attributes)
  end
end


text:
package com.dubture.composer.ui.converter;

import org.apache.commons.lang.StringUtils;
import org.eclipse.core.databinding.conversion.Converter;

import com.dubture.getcomposer.core.collection.JsonArray;

public class Keywords2StringConverter extends Converter {

	public Keywords2StringConverter() {
		super(JsonArray.class, String.class);
	}

	@Override
	public String convert(Object fromObject) {
		if (fromObject == null) {
			return "";
		}
		JsonArray keywords = (JsonArray)fromObject;
		return StringUtils.join((String[])keywords.toArray(new String[]{}), ", ");
	}

}


text:
@implementation UIView (RecursiveDescription)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wundeclared-selector"
-(NSString*)recursiveView
{
    NSString *description = [NSString stringWithFormat:@"%s [Line %d] \r\r %@",
                          __PRETTY_FUNCTION__, __LINE__,
                          [self performSelector:@selector(recursiveDescription)]];
    return description;
}
-(NSString*)constraintsDescription{
    NSString *description = [NSString stringWithFormat:@"%s [Line %d] \r\r %@ \r\r",
                             __PRETTY_FUNCTION__, __LINE__,
                             [[self constraints] description]];
    return description;
}
-(NSString*)autolayoutTraceDescription{
    NSString *description = [NSString stringWithFormat:@"%s [Line %d] \r\r %@",
                             __PRETTY_FUNCTION__, __LINE__,
                             [self performSelector:@selector(_autolayoutTrace)]];
    return description;
}
#pragma clang diagnostic pop
@end


text:
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('general', '0013_auto_20150829_1658'),
    ]

    operations = [
        migrations.AlterField(
            model_name='compmember',
            name='image',
            field=models.ImageField(upload_to='static/images/member_images', help_text='Please select a display image for yourself. This is necessary.'),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='compmember',
            name='name',
            field=models.CharField(max_length=50, help_text='Enter your full name'),
            preserve_default=True,
        ),
    ]


text:
\hypertarget{thread__resource_8cpp}{\section{thread\-\_\-resource.\-cpp File Reference}
\label{thread__resource_8cpp}\index{thread\-\_\-resource.\-cpp@{thread\-\_\-resource.\-cpp}}
}

holds the resource of concurrent unit.

{\ttfamily \#include $<$precompiled.\-h$>$}\\*
{\ttfamily \#include \char`\"{}./thread\-\_\-resource.\-h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}hryky/windows.\-h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}hryky/exclusion.\-h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}hryky/thread/thread\-\_\-callable\-\_\-base.\-h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}hryky/thread/thread\-\_\-pool.\-h\char`\"{}}\\*
{\ttfamily \#include \char`\"{}hryky/chrono.\-h\char`\"{}}\\*

\subsection{Detailed Description}
holds the resource of concurrent unit. \begin{DoxyAuthor}{Author}
H\-R\-Y\-K\-Y
\end{DoxyAuthor}
\begin{DoxyVersion}{Version}

\end{DoxyVersion}
\begin{DoxyParagraph}{Id\-:}
\hyperlink{thread__resource_8cpp}{thread\-\_\-resource.\-cpp} 150 2013-\/02-\/10 10\-:08\-:03\-Z \href{mailto:hryky.private@gmail.com}{\tt hryky.\-private@gmail.\-com}
\end{DoxyParagraph}


text:
title: Courses
type: page

view: 2

header:
  caption: ""
  image: ""
---

I teach the following courses:


text:
package com.gtm.cpims.business.datacollection.submit.reader.jdbc;

import java.lang.reflect.Method;
import java.math.BigDecimal;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import com.gtm.cpims.business.dataapp.statistics.BaseStatisticsService;
import com.gtm.cpims.business.datacollection.submit.ReportSubmitException;
import com.gtm.cpims.business.datacollection.submit.base.BaseReportReader;
import com.gtm.cpims.business.datacollection.submit.base.BaseReportValidator;
import com.gtm.cpims.model.BsSocialsecurityissuessnbyarea;
import com.gtm.cpims.utils.ExcelUtil;

/**
 * 加载金融功能的社会保障卡发卡情况季报表按地区取值保存子类
 *
 * @author Sweet
 *
 */
public class SocialSecurityIssueSSNByAreaReader extends BaseReportReader {

    private JdbcTemplate jdbcTemplate;

    public void setJdbcTemplate(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
    }

    @SuppressWarnings("rawtypes")
    @Transactional(readOnly = false, propagation = Propagation.REQUIRES_NEW)
    protected void read(HSSFWorkbook wb, Map<String, String> reportParam) {
        try {
            jdbcTemplate.execute("DELETE FROM BS_SOCIALSECURITYISSUESSNBYAREA WHERE During = '"
                    + reportParam.get(BaseReportReader.PARAM_DURING) + "' AND bankNo = '"
                    + reportParam.get(BaseReportReader.PARAM_BANKNO) + "'");
        } catch (Exception e) {
        }

        HSSFSheet sheet = wb.getSheetAt(0);
        Class reportDataClass = BsSocialsecurityissuessnbyarea.class;
        Method setMethod = null;
        BigDecimal thisTerm = null;
        BsSocialsecurityissuessnbyarea data = null;
        List<BsSocialsecurityissuessnbyarea> datas = new ArrayList<BsSocialsecurityissuessnbyarea>();
        for (int col = super.config.getBeginColumn(); col <= super.config.getEndColumn(); col++) {
            data = new BsSocialsecurityissuessnbyarea();
            for (int row = super.config.getBeginRow(); row <= super.config.getEndRow(); row++) {
                try {
                    setMethod = reportDataClass.getMethod("setD" + (row + 1), String.class);
                    thisTerm = ExcelUtil.getNumericValue(sheet, row, col);
                    setMethod
                            .invoke(data, String.format(
                                    BaseReportValidator.NUMBER_TO_STRING_PRECISION, thisTerm));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            data.setId(UUID.randomUUID().toString());
            data.setSubmitno(reportParam.get(BaseReportReader.PARAM_SUBMIT_NO));
            data.setBankno(reportParam.get(BaseReportReader.PARAM_BANKNO));
            data.setBankname(reportParam.get(BaseReportReader.PARAM_BANKNAME));
            data.setBanktypename(reportParam.get(BaseReportReader.PARAM_BANKTYPE_NAME));
            data.setBanktypeno(reportParam.get(BaseReportReader.PARAM_BANKTYPE_NO));
            data.setDuring(reportParam.get(BaseReportReader.PARAM_DURING));
            // 获取第4行作为报送机构地区,并去除地区信息空格,保证地区信息统计时一致
            data.setBankarea(ExcelUtil.getValue(sheet, 3, col).trim());
            data.setBankareaindex(super.getArrayIndex(ExcelUtil.getValue(sheet, 3, col).trim(),
                    BaseStatisticsService.AREA_INDEX_ARRAY, true));
            datas.add(data);
        }
        this.insertBatch(datas);
    }

    private static final String INSERT_SQL = "INSERT INTO BS_SOCIALSECURITYISSUESSNBYAREA (ID, BANKNO, BANKNAME, BANKTYPENO "
            + " , BANKTYPENAME, DURING, BANKTYPEINDEX, BANKAREAINDEX, BANKAREA"
            + " , D16, D15, D14, D13, D12, D11, D10, D9, D8, D7, D6, D5, SUBMITNO) "
            + " VALUES (?, ?, ?, ?, ?, ?, null, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";

    public void insertBatch(final List<BsSocialsecurityissuessnbyarea> datas) {
        jdbcTemplate.batchUpdate(INSERT_SQL, new BatchPreparedStatementSetter() {
            public void setValues(PreparedStatement ps, int i) throws SQLException {
                BsSocialsecurityissuessnbyarea data = datas.get(i);
                ps.setString(1, data.getId());
                ps.setString(2, data.getBankno());
                ps.setString(3, data.getBankname());
                ps.setString(4, data.getBanktypeno());
                ps.setString(5, data.getBanktypename());
                ps.setString(6, data.getDuring());
                ps.setInt(7, data.getBankareaindex());
                ps.setString(8, data.getBankarea());
                ps.setString(9, data.getD16());
                ps.setString(10, data.getD15());
                ps.setString(11, data.getD14());
                ps.setString(12, data.getD13());
                ps.setString(13, data.getD12());
                ps.setString(14, data.getD11());
                ps.setString(15, data.getD10());
                ps.setString(16, data.getD9());
                ps.setString(17, data.getD8());
                ps.setString(18, data.getD7());
                ps.setString(19, data.getD6());
                ps.setString(20, data.getD5());
                ps.setString(21, data.getSubmitno());
            }

            public int getBatchSize() {
                return datas.size();
            }
        });
    }

    /**
     * 删除已经存在该所属期数据
     *
     * @param reportParam
     * @throws ReportSubmitException
     */
    @Transactional(readOnly = false)
    protected void deleteExsitData(Map<String, String> reportParam) throws ReportSubmitException {
    }
}


text:
package org.whitehole.infra.json;

public class JsonException extends RuntimeException {

	private static final long serialVersionUID = -1957349673491618346L;

	private final String _message;
	private final Throwable _cause;

	public JsonException(String message) {
		_message = message;
		_cause = null;
	}

	public JsonException(String message, Throwable cause) {
		_message = message;
		_cause = cause;
	}

	public String getMessage() {
		return _message;
	}

	public Throwable getCause() {
		return _cause;
	}

}


text:
import random
import zipfile
import gzip
import numpy as np
from srsly import cloudpickle as pickle

from ..util import unzip, to_categorical, get_file
from .._registry import register_loader

MNIST_URL = "https://s3.amazonaws.com/img-datasets/mnist.pkl.gz"
EMNIST_URL = "http://www.itl.nist.gov/iaui/vip/cs_links/EMNIST/gzip.zip"
EMNIST_FILE = "gzip.zip"

FA_TRAIN_IMG_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz"
FA_TRAIN_LBL_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz"
FA_TEST_IMG_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz"
FA_TEST_LBL_URL = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz"

KU_TRAIN_IMG_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-imgs.npz"
KU_TRAIN_LBL_URL = (
    "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-train-labels.npz"
)
KU_TEST_IMG_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-imgs.npz"
KU_TEST_LBL_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/kmnist-test-labels.npz"

@register_loader("mnist")
def mnist(variant="mnist", shuffle=True):
    if variant == "mnist":
        (X_train, y_train), (X_test, y_test) = load_mnist()
    elif variant == "fashion":
        (X_train, y_train), (X_test, y_test) = load_fashion_mnist()
    elif variant == "kuzushiji":
        (X_train, y_train), (X_test, y_test) = load_kuzushiji_mnist()
    elif variant.startswith("emnist-"):
        if len(variant.split("-")) != 2:
            raise ValueError(
                "EMNIST data set should be given in format "
                "'emnist-subset', where 'subset' can be "
                "'digits', 'letters', 'balanced' "
                "'byclass', 'bymerge' and 'mnist'. "
                f"{variant} was provided."
            )
        subset = variant.split("-")[1]
        if subset not in [
            "digits",
            "letters",
            "balanced",
            "byclass",
            "bymerge",
            "mnist",
        ]:
            raise ValueError(
                "To load EMNIST use the format "
                "'emnist-subset' where 'subset' can be"
                "'digits', 'letters', 'balanced' "
                "'byclass', 'bymerge' and 'mnist'."
            )
        else:
            (X_train, y_train), (X_test, y_test) = load_emnist(subset=subset)
    else:
        raise ValueError(
            "Variant must be one of: "
            "'mnist', 'fashion', 'kuzushiji', "
            "'emnist-digits', 'emnist-letters', "
            "'emnist-balanced', 'emnist-byclass', "
            "'emnist-bymerge', 'emnist-mnist'."
        )
    n_train = X_train.shape[0]
    n_test = X_test.shape[0]
    n_classes = len(np.unique(y_train))
    X_train = X_train.reshape(n_train, 784)
    X_test = X_test.reshape(n_test, 784)
    X_train = X_train.astype("float32")
    X_test = X_test.astype("float32")
    X_train /= 255.0
    X_test /= 255.0
    if shuffle:
        train_data = list(zip(X_train, y_train))
        random.shuffle(train_data)
        X_train, y_train = unzip(train_data)
    y_train = to_categorical(y_train, n_classes=n_classes)
    y_test = to_categorical(y_test, n_classes=n_classes)
    return (X_train, y_train), (X_test, y_test)

def load_mnist(path="mnist.pkl.gz"):
    path = get_file(path, origin=MNIST_URL)
    if path.endswith(".gz"):
        f = gzip.open(path, "rb")
    else:
        f = open(path, "rb")
    data = pickle.load(f, encoding="bytes")
    f.close()
    return data  # (X_train, y_train), (X_test, y_test)

def load_fashion_mnist(
    train_img_path="train-images-idx3-ubyte.gz",
    train_label_path="train-labels-idx1-ubyte.gz",
    test_img_path="t10k-images-idx3-ubyte.gz",
    test_label_path="t10k-labels-idz1-ubyte.gz",
):
    train_img_path = get_file(train_img_path, origin=FA_TRAIN_IMG_URL)
    train_label_path = get_file(train_label_path, origin=FA_TRAIN_LBL_URL)
    test_img_path = get_file(test_img_path, origin=FA_TEST_IMG_URL)
    test_label_path = get_file(test_label_path, origin=FA_TEST_LBL_URL)
    # Based on https://github.com/zalandoresearch/fashion-mnist/blob/master/utils/mnist_reader.py
    with gzip.open(train_label_path, "rb") as trlbpath:
        train_labels = np.frombuffer(trlbpath.read(), dtype=np.uint8, offset=8)
    with gzip.open(train_img_path, "rb") as trimgpath:
        train_images = np.frombuffer(
            trimgpath.read(), dtype=np.uint8, offset=16
        ).reshape(len(train_labels), 28, 28)
    with gzip.open(test_label_path, "rb") as telbpath:
        test_labels = np.frombuffer(telbpath.read(), dtype=np.uint8, offset=8)
    with gzip.open(test_img_path, "rb") as teimgpath:
        test_images = np.frombuffer(
            teimgpath.read(), dtype=np.uint8, offset=16
        ).reshape(len(test_labels), 28, 28)
    return (train_images, train_labels), (test_images, test_labels)

def load_kuzushiji_mnist(
    train_img_path="kmnist-train-imgs.npz",
    train_label_path="kmnist-train-labels.npz",
    test_img_path="kmnist-test-imgs.npz",
    test_label_path="kmnist-test-labels.npz",
):
    train_img_path = get_file(train_img_path, origin=KU_TRAIN_IMG_URL)
    train_label_path = get_file(train_label_path, origin=KU_TRAIN_LBL_URL)
    test_img_path = get_file(test_img_path, origin=KU_TEST_IMG_URL)
    test_label_path = get_file(test_label_path, origin=KU_TEST_LBL_URL)
    train_images = np.load(train_img_path)["arr_0"]
    train_labels = np.load(train_label_path)["arr_0"]
    test_images = np.load(test_img_path)["arr_0"]
    test_labels = np.load(test_label_path)["arr_0"]
    return (train_images, train_labels), (test_images, test_labels)

def _decode_idx(archive, path):
    comp = archive.read(path)
    data = bytes(gzip.decompress(comp))
    axes = data[3]
    shape = []
    dtype = np.dtype("ubyte").newbyteorder(">")
    for axis in range(axes):
        offset = 4 * (axis + 1)
        size = int(np.frombuffer(data[offset : offset + 4], dtype=">u4"))
        shape.append(size)
    shape = tuple(shape)
    offset = 4 * (axes + 1)
    flat = np.frombuffer(data[offset:], dtype=dtype)
    reshaped = flat.reshape(shape)
    return reshaped

def load_emnist(path=EMNIST_FILE, subset="digits"):
    emnist_path = get_file(path, origin=EMNIST_URL)
    train_X_path = f"gzip/emnist-{subset}-train-images-idx3-ubyte.gz"
    train_y_path = f"gzip/emnist-{subset}-train-labels-idx1-ubyte.gz"
    test_X_path = f"gzip/emnist-{subset}-test-images-idx3-ubyte.gz"
    test_y_path = f"gzip/emnist-{subset}-test-labels-idx1-ubyte.gz"
    with zipfile.ZipFile(emnist_path, "r") as archive:
        train_X = _decode_idx(archive, train_X_path)
        train_y = _decode_idx(archive, train_y_path)
        test_X = _decode_idx(archive, test_X_path)
        test_y = _decode_idx(archive, test_y_path)
        # For some reason in this data set the labels start from 1
        if subset == "letters":
            train_y = train_y - 1
            test_y = test_y - 1
    return (train_X, train_y), (test_X, test_y)


text:
ROOT = ../../../..
include $(ROOT)/mak/config.mak
include $(ROOT)/test/config.mak

XMLSCRIPTS = DeleteClass36005

include $(ROOT)/test/configend.mak


text:
package drones.scheduler.messages.to;

/**
 * Created by Ronald on 18/04/2015.
 */
public class SchedulerRequestMessage {

    private static long count = 0;
    private final long requestId;

    public SchedulerRequestMessage() {
        requestId = count;
        count++;
    }

    public long getRequestId() {
        return requestId;
    }
}


text:
import test from 'ava'
import { readFile, writeFile, find, rm } from 'fildes-extra'
import { join } from 'path'
import imagemin from 'imagemin'
import imageminZopfli from 'imagemin-zopfli'

const images = join(__dirname, 'images')
const build = join(__dirname, 'build')

test.before(() => {
  return find(join(build, '**/zopfli.*.png'))
    .then(files => files.map(file => rm(file)))
})

const zopfli = files => {
  return Promise.all(files.map(file => {
    return readFile(join(images, file))
      .then(buffer => imagemin.buffer(buffer, {
        plugins: [imageminZopfli({
          // 8bit: false,
          // transparent: false,
          // iterations: 15,
          // iterationsLarge: 5,
          // more: false,
        })]
      }))
      .then(buffer => writeFile(join(build, file, 'zopfli.default.png'), buffer))
  }))
}

test('zopfli', t => {
  return find('*.png', { cwd: images })
    .then(zopfli)
    .then(() => find(join(build, '**/zopfli.*.png')))
    .then(imgs => t.truthy(imgs.length, `found ${imgs.length} zopfli's`))
})


text:
package k57ca.pmp.askeverywhere;

import javax.persistence.EntityManagerFactory;
import javax.persistence.Persistence;

public final class EMF {
	private static final EntityManagerFactory emfInstance = Persistence
			.createEntityManagerFactory("transactions-optional");

	private EMF() {
	}

	public static EntityManagerFactory get() {
		return emfInstance;
	}
}


text:
package picard.sam;

import htsjdk.samtools.fastq.FastqReader;
import htsjdk.samtools.fastq.FastqRecord;
import htsjdk.samtools.util.IOUtil;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import picard.PicardException;
import picard.cmdline.CommandLineProgramTest;

import java.io.File;
import java.io.IOException;
import java.util.*;

public class SamToFastqWithTagsTest extends CommandLineProgramTest {
    private static final File TEST_DATA_DIR = new File("testdata/picard/sam/bam2fastq/paired");

    public String getCommandLineProgramName() {
        return SamToFastqWithTags.class.getSimpleName();
    }

    private static final String[] STG_ARRAY = {"UR,CR", "CR,UR"};
    @DataProvider(name = "argumentCombinationsdata")
    public Object[][] argumentCombinationsdata() {
        return new Object[][]{
                // not supplying any arguments
                {null, null, null, 1},
                // only supply STGs and no QTG or SEP
                {STG_ARRAY, null, null, 0},
                // not supplying the correct number of QTGs or SEPs
                {STG_ARRAY, new String[]{"UY"}, null, 1},
                {STG_ARRAY, null, new String[]{"AAAAA"}, 1},
                // supplying correct number of QTGs or SEPs
                {new String[]{"UR,CR"}, new String[]{"CY,UY"}, null, 0},
                {STG_ARRAY, null, new String[]{"AAA", "AAAAA"}, 0},
                {STG_ARRAY, new String[]{"CY", "CY,UY"}, new String[]{"AAA", "AAAAA"}, 0},

        };
    }

    @Test(dataProvider = "argumentCombinationsdata")
    public void argumentCombinations(final String[] stgArguments,
                                     final String[] qtgArguments,
                                     final String[] sepArguments,
                                     final int returnCode) throws IOException {

        final File temp_input = File.createTempFile("input", ".fastq");
        temp_input.deleteOnExit();
        final File temp_input_2 = File.createTempFile("input_2", ".fastq");
        temp_input_2.deleteOnExit();
        final File temp_cr = File.createTempFile("UR_CR", ".fastq");
        temp_cr.deleteOnExit();
        final File temp_cr_ur = File.createTempFile("CR_UR", ".fastq");
        temp_cr_ur.deleteOnExit();

        final ArrayList<String> args = new ArrayList<>();
        args.add("INPUT=" + TEST_DATA_DIR + "/ok/sorted-pair.sam");
        args.add("FASTQ=" + temp_input.getAbsolutePath());
        args.add("SECOND_END_FASTQ=" + temp_input_2.getAbsolutePath());

        if (stgArguments != null) {
            for (String value : stgArguments) {
                args.add("STG=" + value);
            }
        }

        if (qtgArguments != null) {
            for (String value : qtgArguments) {
                args.add("QTG=" + value);
            }
        }

        if (sepArguments != null) {
            for (String value : sepArguments) {
                args.add("SEP=" + value);
            }
        }

        // make sure program invocation returns the expected return code
        Assert.assertEquals(runPicardCommandLine(args), returnCode);
    }

    @DataProvider(name = "queryNonExistantTag")
    public Object[][] queryNonExistantTag() {
        return new Object[][] {
                {"ok/sorted-single.sam"}
        };
    }

    @Test (dataProvider = "queryNonExistantTag", expectedExceptions =  PicardException.class)
    public void testQueryNonExistantTag(final String samFilename) throws IOException {
        final File temp_input = File.createTempFile("input", ".fastq");
        temp_input.deleteOnExit();

        runPicardCommandLine(new String[]{
                "INPUT=" + TEST_DATA_DIR + "/" + samFilename,
                "FASTQ=" + temp_input.getAbsolutePath(),
                "STG=BR"
        });
    }

    @DataProvider(name = "okGroupedFiles")
    public Object[][] okGroupedFiles() {
        return new Object[][] {
                {"ok/grouped-last-pair-mates-flipped.sam", new String[]{"rg1","rg2"}},
        };
    }

    // make sure the interleaved fastq and sequence tag fastq contain the same mate pairs
    @Test(dataProvider = "okGroupedFiles")
    public void testOkGroupedFiles(final String samFilename, final String [] groupFiles) throws IOException {
        final File samFile = new File(TEST_DATA_DIR,samFilename);
        final Map<String, Set<String>> outputSets = new HashMap<>(groupFiles.length);

        final String tmpDir = IOUtil.getDefaultTmpDir().getAbsolutePath() + "/";
        final String [] args = {
                "INPUT=" + samFile.getAbsolutePath(),
                "OUTPUT_PER_RG=true",
                "OUTPUT_DIR=" + tmpDir,
                "INTERLEAVE=true",
                "STG=CR,UR",
                "QTG=CY,UY"
        };
        runPicardCommandLine(args);

        Set<String> outputHeaderSet1;
        Set<String> outputHeaderSet2;
        for (final String groupPUName : groupFiles) {
            String keyName1 = groupPUName + "_1";
            String keyName2 = groupPUName + "_CR_UR";
            String fname1 = tmpDir + "/" + keyName1 + ".fastq";
            String fname2 = tmpDir + "/" + keyName2 + ".fastq";
            File f1 = new File(fname1);
            File f2 = new File(fname2);
            f1.deleteOnExit();
            f2.deleteOnExit();
            IOUtil.assertFileIsReadable(f1);
            IOUtil.assertFileIsReadable(f2);

            // Check that paired fastq files are same size and store them for later comparison
            outputHeaderSet1 = SamToFastqTest.createFastqReadHeaderSet(f1);
            outputHeaderSet2 = SamToFastqTest.createFastqReadHeaderSet(f2);
            outputSets.put(keyName1 , outputHeaderSet1);
            outputSets.put(keyName2, outputHeaderSet2);
            Assert.assertEquals(outputHeaderSet1.size(), outputHeaderSet2.size());
        }

        // Create map of read groups and mate pairs from SAM records
        final Map<String, Map<String,SamToFastqTest.MatePair>> map = SamToFastqTest.createPUPairsMap(samFile);

        for(final Map.Entry<String, Map<String, SamToFastqTest.MatePair>> groupEntry : map.entrySet()) {
            // Ensure that for each group, each mate of each pair in the SAM file is in the correct fastq pair file
            for (final Map.Entry<String,SamToFastqTest.MatePair> entry : groupEntry.getValue().entrySet() ) {
                final SamToFastqTest.MatePair mpair = entry.getValue();
                outputHeaderSet1 = outputSets.get(groupEntry.getKey() + "_1");
                outputHeaderSet2 = outputSets.get(groupEntry.getKey() + "_CR_UR");

                Assert.assertNotNull(mpair.mate1); // ensure we have two mates
                Assert.assertNotNull(mpair.mate2);
                Assert.assertEquals(mpair.mate1.getReadName(),mpair.mate2.getReadName());
                final String readName = mpair.mate1.getReadName() ;
                Assert.assertTrue(outputHeaderSet1.contains(readName + "/1")); // ensure mate is in correct file
                Assert.assertTrue(outputHeaderSet2.contains(readName + "/2"));
            }
        }
    }

    @DataProvider(name = "testTagGroupFastqData")
    public Object[][] testTagGroupFastqData() {
        return new Object[][] {
                {new String[]{"CR"}, new String[]{"CY"}, null, "AAAAA", "11111", "CCCCC", "22222", null, null, null, null, "One Sequence Tag Group"},
                {new String[]{"CR,UR"}, new String[]{"CY,UY"}, null, "AAAAATTT", "11111222", "CCCCCGGG", "22222111", null, null, null, null, "One Sequence Tag Group with Two Tags"},
                {new String[]{"CR,UR", "UR,CR"}, new String[]{"CY,UY", "UY,CY"}, null, "AAAAATTT", "11111222", "CCCCCGGG", "22222111", "TTTAAAAA", "22211111", "GGGCCCCC", "11122222", "Two Sequence Tag Groups with Two Tags"},
                {new String[]{"CR,UR", "UR,CR"}, new String[]{"CY,UY", "UY,CY"}, new String[]{"ATGC", "CGTA"} , "AAAAAATGCTTT", "11111~~~~222", "CCCCCATGCGGG", "22222~~~~111", "TTTCGTAAAAAA", "222~~~~11111", "GGGCGTACCCCC", "111~~~~22222", "Two Sequence Tag Groups with Two Tags with Separators"},

        };
    }

    @Test(dataProvider = "testTagGroupFastqData")
    public void testTagGroupFastq(final String[] sequenceTags, final String[] qualityTags, final String[] sepArgs, final String bases_1_1, final String quals_1_1,
                             final String bases_1_2, final String quals_1_2, final String bases_2_1, final String quals_2_1, final String bases_2_2,
                             final String quals_2_2, final String testName) throws IOException {
        final File samFile = new File(TEST_DATA_DIR, "ok/sorted-single.sam") ;
        final File fastq = File.createTempFile("testtag", "fastq");
        final File f1 = new File(fastq.getParent(), sequenceTags[0].replace(",", "_") + ".fastq");
        File f2 = null;
        if (quals_2_1 != null) {
            f2 = new File(fastq.getParent(), sequenceTags[1].replace(",", "_") + ".fastq");
            f2.deleteOnExit();
        }
        f1.deleteOnExit();

        final ArrayList<String> args = new ArrayList<>();
        args.add("INPUT=" + samFile.getAbsolutePath());
        args.add("FASTQ=" + fastq.getAbsolutePath());

        for (String value : sequenceTags) {
            args.add("STG=" + value);
        }
        if (qualityTags != null) {
            for (String value : qualityTags) {
                args.add("QTG=" + value);
            }
        }
        if (sepArgs != null) {
            for (String value : sepArgs) {
                args.add("SEP=" + value);
            }
        }

        Assert.assertEquals(runPicardCommandLine(args), 0);

        Iterator<FastqRecord> it = new FastqReader(f1).iterator();
        FastqRecord first = it.next();
        Assert.assertEquals(first.getReadString(), bases_1_1, testName);
        Assert.assertEquals(first.getBaseQualityString(), quals_1_1, testName);
        FastqRecord second = it.next();
        Assert.assertEquals(second.getReadString(), bases_1_2, testName);
        Assert.assertEquals(second.getBaseQualityString(), quals_1_2, testName);
        if (bases_2_1 != null) {
            it = new FastqReader(f2).iterator();
            first = it.next();
            Assert.assertEquals(first.getReadString(), bases_2_1, testName);
            Assert.assertEquals(first.getBaseQualityString(), quals_2_1, testName);
            second = it.next();
            Assert.assertEquals(second.getReadString(), bases_2_2, testName);
            Assert.assertEquals(second.getBaseQualityString(), quals_2_2, testName);
        }
    }
}


text:
CREATE TABLE IF NOT EXISTS `#__wf_profiles` (
    `id` int(11) NOT NULL AUTO_INCREMENT,
    `name` varchar(255) NOT NULL,
    `description` text NOT NULL,
    `users` text NOT NULL,
    `types` text NOT NULL,
    `components` text NOT NULL,
    `area` tinyint(3) NOT NULL,
    `device` varchar(255) NOT NULL,
    `rows` text NOT NULL,
    `plugins` text NOT NULL,
    `published` tinyint(3) NOT NULL,
    `ordering` int(11) NOT NULL,
    `checked_out` tinyint(3) NOT NULL,
    `checked_out_time` datetime NOT NULL,
    `params` text NOT NULL,
    PRIMARY KEY (`id`)
) DEFAULT CHARSET=utf8;


text:
package de.hdodenhof.xingapi.internal.response;

import de.hdodenhof.xingapi.models.Tags;

public class TagsWrapper implements ResponseWrapper<Tags> {

    Tags tags;

    @Override
    public Tags unwrap() {
        return tags;
    }
}


text:
def removeEffectLessNode(target, Q):
	pass

def findSearchSequence(target):
	pass

def Elim(target, Z):
	pass

def VE(target, E, e, Q, p):
	pass

def VE_MAP(target, p):
	pass


text:
package io.feellix.restvideodemo.domain.video;

import io.feellix.restvideodemo.common.ApplicationService;
import org.springframework.core.convert.ConversionService;
import org.springframework.web.multipart.MultipartFile;

import java.util.List;

import static java.util.stream.Collectors.toList;

@ApplicationService
public class VideoService {

    private final VideoFactory videoFactory;
    private final VideoRepository videoRepository;
    private final ConversionService conversionService;

    public VideoService(VideoFactory videoFactory, VideoRepository videoRepository, ConversionService conversionService) {
        this.videoFactory = videoFactory;
        this.videoRepository = videoRepository;
        this.conversionService = conversionService;
    }

    public List<VideoDTO> getAllVideos(){
        return videoRepository.findAll()
                .stream()
                .map(videoEntity -> conversionService.convert(videoEntity, VideoDTO.class))
                .collect(toList());
    }

    public VideoDTO getOne(Long id) {
        return conversionService.convert(videoRepository.getOne(id), VideoDTO.class);
    }

    public VideoDTO createFrom(MultipartFile file) {
        VideoEntity videoEntity = videoFactory.createFrom(file);
        videoRepository.save(videoEntity);
        return conversionService.convert(videoEntity, VideoDTO.class);
    }
}


text:
<?php

namespace Ikodota\Yeoman\Providers;

use Ikodota\Yeoman\Models\Permission;
use Illuminate\Support\Facades\Gate;
use Illuminate\Foundation\Support\Providers\AuthServiceProvider as ServiceProvider;

class AuthServiceProvider extends ServiceProvider
{
    /**
     * The policy mappings for the application.
     *
     * @var array
     */
    protected $policies = [
        'App\Model' => 'App\Policies\ModelPolicy',
    ];

    /**
     * Register any authentication / authorization services.
     *
     * @return void
     */
    public function boot()
    {
        $this->registerPolicies();

        foreach ($this->getPermissions() as $permission){
            Gate::define($permission->name,function($user) use ($permission){
                if($user->isSuperAdmin() || $user->hasPermission($permission)){
                    return true;
                }
            });
        }
    }

    public function getPermissions()
    {
        try{
            return Permission::with('roles')->get();//拿到所有的permissions和对应的roles
        }catch (\Exception $exception){
            return;
        }

    }
}


text:
<?php

namespace Google\Service\CloudRun;

class GoogleCloudRunV2EnvVarSource extends \Google\Model
{
  protected $secretKeyRefType = GoogleCloudRunV2SecretKeySelector::class;
  protected $secretKeyRefDataType = '';

  /**
   * @param GoogleCloudRunV2SecretKeySelector
   */
  public function setSecretKeyRef(GoogleCloudRunV2SecretKeySelector $secretKeyRef)
  {
    $this->secretKeyRef = $secretKeyRef;
  }
  /**
   * @return GoogleCloudRunV2SecretKeySelector
   */
  public function getSecretKeyRef()
  {
    return $this->secretKeyRef;
  }
}

// Adding a class alias for backwards compatibility with the previous class name.
class_alias(GoogleCloudRunV2EnvVarSource::class, 'Google_Service_CloudRun_GoogleCloudRunV2EnvVarSource');


text:
from django.test import TestCase
from comp.models import Comp

class CompTestCase(TestCase):
    def setUp(self):
        for i in range(100):
            Comp(name='comp%s' % i, alias='c%s' % i, revision='A').save()


text:
using System.Collections.Generic;

using Amazon.ElasticBeanstalk.Model;
using Amazon.Runtime.Internal.Transform;

namespace Amazon.ElasticBeanstalk.Model.Internal.MarshallTransformations
{
     /// <summary>
     ///   ValidationMessage Unmarshaller
     /// </summary>
    internal class ValidationMessageUnmarshaller : IUnmarshaller<ValidationMessage, XmlUnmarshallerContext>
    {
        public ValidationMessage Unmarshall(XmlUnmarshallerContext context)
        {
            ValidationMessage validationMessage = new ValidationMessage();
            int originalDepth = context.CurrentDepth;
            int targetDepth = originalDepth + 1;
            if (context.IsStartOfDocument)
               targetDepth += 2;

            while (context.Read())
            {
                if (context.IsStartElement || context.IsAttribute)
                {
                    if (context.TestExpression("Message", targetDepth))
                    {
                        validationMessage.Message = StringUnmarshaller.GetInstance().Unmarshall(context);

                        continue;
                    }
                    if (context.TestExpression("Severity", targetDepth))
                    {
                        validationMessage.Severity = StringUnmarshaller.GetInstance().Unmarshall(context);

                        continue;
                    }
                    if (context.TestExpression("Namespace", targetDepth))
                    {
                        validationMessage.Namespace = StringUnmarshaller.GetInstance().Unmarshall(context);

                        continue;
                    }
                    if (context.TestExpression("OptionName", targetDepth))
                    {
                        validationMessage.OptionName = StringUnmarshaller.GetInstance().Unmarshall(context);

                        continue;
                    }
                }
                else if (context.IsEndElement && context.CurrentDepth < originalDepth)
                {
                    return validationMessage;
                }
            }

            return validationMessage;
        }

        private static ValidationMessageUnmarshaller instance;

        public static ValidationMessageUnmarshaller GetInstance()
        {
            if (instance == null)
               instance = new ValidationMessageUnmarshaller();

            return instance;
        }
    }
}


text:
package com.github.tonivade.claudb.command.key;

import static com.github.tonivade.claudb.DatabaseValueMatchers.nullValue;
import static com.github.tonivade.claudb.data.DatabaseValue.string;
import static org.hamcrest.CoreMatchers.is;

import org.junit.Rule;
import org.junit.Test;

import com.github.tonivade.resp.protocol.RedisToken;
import com.github.tonivade.claudb.command.CommandRule;
import com.github.tonivade.claudb.command.CommandUnderTest;

@CommandUnderTest(DeleteCommand.class)
public class DeleteCommandTest {

  @Rule
  public final CommandRule rule = new CommandRule(this);

  @Test
  public void testExecute() {
    rule.withData("test", string("value"))
    .withParams("test")
    .execute()
    .assertValue("test", is(nullValue()))
    .assertThat(RedisToken.integer(1));
  }

}


text:
package com.temportalist.weepingangels.common.block

import java.util

import com.temportalist.origin.api.common.block.BlockTile
import com.temportalist.origin.api.common.lib.{BlockState, V3O}
import com.temportalist.origin.api.common.utility.Stacks
import com.temportalist.weepingangels.common.tile.TEStatue
import com.temportalist.weepingangels.common.{WAOptions, WeepingAngels}
import net.minecraft.block.Block
import net.minecraft.block.material.Material
import net.minecraft.entity.EntityLivingBase
import net.minecraft.entity.player.EntityPlayer
import net.minecraft.init.Blocks
import net.minecraft.item.ItemStack
import net.minecraft.tileentity.TileEntity
import net.minecraft.world.{IBlockAccess, World}

/**
 *
 *
 * @author TheTemportalist
 */
class BlockStatue(material: Material, name: String, teclass: Class[_ <: TileEntity])
		extends BlockTile(material, WeepingAngels.MODID, name, teclass) {

	val maxHeight: Int = 3
	val maxHeight_Bound: Float = 2.5F

	this.setHardness(2.0F)
	this.setResistance(10.0F)
	this.setStepSound(Block.soundTypeStone)

	override def isOpaqueCube: Boolean = false

	override def getRenderType: Int = -1

	override def canPlaceBlockAt(world: World, x: Int, y: Int, z: Int): Boolean = {
		for (y1 <- 0 until this.maxHeight)
			if (world.getBlock(x, y + y1, z) != Blocks.air) return false
		true
	}

	override def onBlockAdded(worldIn: World, x: Int, y: Int, z: Int): Unit = {
		if (worldIn.getBlockMetadata(x, y, z) == 0) {
			for (y1: Int <- 1 until this.maxHeight) {
				worldIn.setBlock(x, y + y1, z, this, y1, 3)
			}
			if (worldIn.isRemote)
				worldIn.getTileEntity(x, y, z).asInstanceOf[TEStatue].refreshTexture()
		}
	}

	override def setBlockBoundsBasedOnState(world: IBlockAccess, x: Int, y: Int, z: Int): Unit = {
		val meta: Int = world.getBlockMetadata(x, y, z)
		this.setBlockBounds(0F, 0F - meta, 0F, 1F, this.maxHeight_Bound - meta, 1F)
	}

	override def breakBlock(world: World, x: Int, y: Int, z: Int, block: Block, meta: Int): Unit = {
		if (meta == 0) {
			for (y1 <- 1 to meta) world.setBlockToAir(x, y - y1, z)
			for (y1 <- 1 until this.maxHeight - meta) world.setBlockToAir(x, y + y1, z)
			world.removeTileEntity(x, y, z)
		}
	}

	override def getDrops_Pre(world: World, pos: V3O, state: BlockState,
			tile: TileEntity): util.List[ItemStack] = {
		if (state.getMeta == 0) super.getDrops_Pre(world, pos, state, tile)
		else new util.ArrayList[ItemStack]()
	}

	override def onBlockPlacedBy(world: World, x: Int, y: Int, z: Int, placer: EntityLivingBase,
			stack: ItemStack): Unit = {
		world.getTileEntity(x, y, z) match {
			case tile: TEStatue =>
				var rotation: Float = placer.rotationYaw
				if (placer.isSneaking) rotation = 360 - rotation
				tile.setRotation(rotation)
			case _ =>
		}
	}

	override def onBlockActivated(worldIn: World, x: Int, y: Int, z: Int, player: EntityPlayer,
			side: Int, subX: Float, subY: Float, subZ: Float): Boolean = {
		if (!player.isSneaking) {
			new V3O(x, y, z).down(worldIn.getBlockMetadata(x, y, z))
					.openGui(WeepingAngels, WAOptions.statueGui, player)
			true
		}
		else
			false
	}

}


text:
layout: post
title: "One week of French election tweets"
description: "A ton of text to test readability."
tags: [data science, election]
categories: [data science]
jsarr:
- graphs/test_graph.js
---

This is a quick showcase of some data I collected for one week during the French presidential primaries for the republicans. For this I used my raspi-based spark cluster to stream tweets relating to the election. Here I show some scala code to process this dataset to create interesting network visualizations out of it.

## Processing of tweets using scala

First define nodes, edges and weighted edges as case clases:

{% highlight scala %}

// tweet hashtags are just a list of hashtag strings
type TweetHashtags = List[String]

// A node is an object with a label and a weight
// Two nodes with the same label are the same
case class Node(label: String, weight: Int) {
    def ==(that: Node): Boolean = this.label == that.label
}

// An edge is an object with a source node and a target node
case class Edge(source: Node, target: Node) extends Ordered[Edge] {
    def sorted = if (this.source.label < this.target.label) Edge(source, target) else Edge(target, source)

    def compare(that: Edge) = this.source.label compare that.source.label

    // an edge from A -> B is the same as an edge from B -> A
    def ==(that: Edge): Boolean = (
        ((this.source.label == that.source.label)&&(this.target.label == that.target.label))
        ||((this.source.label == that.target.label)&&(this.target.label == that.source.label))
      )
}

// A weighted edge is similar but also has an int weight on its edge
case class WeightedEdge(source: Node, target: Node, weight: Int) extends Ordered[WeightedEdge] {
    def sorted = if (this.source.label < this.target.label) Edge(source, target) else Edge(target, source)

    def compare(that: WeightedEdge) = this.weight compare that.weight

    def ==(that: Edge): Boolean = (
        (this.source == that.source)&&(this.target == that.target))
        ||((this.source == that.target)&&(this.target == that.source))
}

{% endhighlight %}

Bunch of functions to create a network from the collected tweets

{% highlight scala %}
import scala.annotation.tailrec

// create a list of edges from a single tweet's hashtags
def singleTweetEdges(tweet: List[String]): List[Edge] = {
    tweet match {
      case Nil => List()
      case List(x) => List()
      case x :: y :: ys => (List(Edge(Node(x, 1), Node(y, 1)).sorted) ::: singleTweetEdges(x :: ys) ::: singleTweetEdges(y :: ys)).distinct
    }
  }

// create a list of edges from a list of tweets
def getEdgeList(tweetList: List[List[String]]): List[Edge] = {
    tweetList match {
      case Nil => List()
      case List() => List()
      case tweet :: ys => {
        ys match {
          case y :: ys1 => singleTweetEdges(tweet) ::: getEdgeList(ys)
          case Nil => singleTweetEdges(tweet)
        }
      }
    }
  }

def containsNode(list: List[Node], newNode: Node): Boolean =
    list.exists(node => node.label == newNode.label)

def containsEdge(list: List[WeightedEdge], newEdge: WeightedEdge): Boolean =
    list.exists(we => (we.source.label == newEdge.source.label)&&(we.target.label == newEdge.target.label))

def countEdges(list: List[Edge]): List[(Edge, Int)] = {
    list.map{case Edge(source, target) => (Edge(source, target).sorted, 1)}.groupBy(pair => pair._1).mapValues(_.length).toList
    }

def insertEdges(edges: List[Edge], acc: List[WeightedEdge]): List[WeightedEdge] = {
    edges match {
      case List() => acc
      case y :: ys => {
        val ySorted = y.sorted
        if (containsEdge(acc, WeightedEdge(ySorted.source, ySorted.target, 1))) {
          val inserted = for (we <- acc) yield {
            if (we == ySorted) WeightedEdge(ySorted.source, ySorted.target, 1 + we.weight)
            else we
        }
          insertEdges(ys, inserted)
        }
        else insertEdges(ys, acc ::: List(WeightedEdge(ySorted.source, ySorted.target, 1)))
      }
    }
    }

def updateWeightedEdges(tweet: TweetHashtags, acc: List[WeightedEdge]): List[WeightedEdge] = {
    val newEdges = singleTweetEdges(tweet)
    insertEdges(newEdges, acc)
    }

def updateNodesFromHashtag(ht: String, nodeList: List[Node]): List[Node] = {
    if (containsNode(nodeList, Node(ht, 1))) {
      for (node <- nodeList) yield {
        if (node.label == ht) Node(ht, 1 + node.weight)
        else node
      }
    }
    else nodeList ::: List(Node(ht, 1))
    }

def updateNodesFromTweet(tweet: TweetHashtags, acc: List[Node]): List[Node] =
    tweet match {
      case Nil => acc
      case y :: ys => updateNodesFromTweet(ys, updateNodesFromHashtag(y, acc))
    }

@tailrec
final def createHashtagNetAcc(tweetList: List[TweetHashtags], acc: (List[WeightedEdge], List[Node])): (List[WeightedEdge], List[Node]) =
    tweetList match {
      case Nil => acc
      case y :: ys => createHashtagNetAcc(ys, (updateWeightedEdges(y, acc._1), updateNodesFromTweet(y, acc._2)))
    }

def createHashtagNetwork(tweetList: List[TweetHashtags]): (List[WeightedEdge], List[Node]) =
    createHashtagNetAcc(tweetList, (List(), List()))
{% endhighlight %}

Then we can simply create the network with the following:

{% highlight scala %}
// read in the files I saved in parquet format
val data = spark.read.parquet("/path/to/parquet/files/*")

// convert the tweets to hashtag lists
val hashtagsByTweet = data.map((x: Row) => x.getString(5).split(" ").filter((str: String) => str.startsWith("#")).map((str: String) => str.toLowerCase.replaceAll("[^\\p{IsAlphabetic}^\\p{IsDigit}^@^#]", "")))

def listified = hashtagsByTweet.collect().map(x => x.toList).toList
// then create the network from the list of tweets
val (weightedEdges, weightedNodes) = createHashtagNetwork(listified)
{% endhighlight %}

## Results

To visualise how people are talking about the election, one intuitive way is to use physics. Here we take the nodes of the networks (either individual hashtags, or common mentioned twitter handles) and have them interact with other nodes via some interaction, modeling a forces between the nodes which are attractive if the two nodes have a weighty edge between them and slightly repulsive otherwise. We can use vis.js to generate some pretty cool intuitive visualisations of the conversation on twitter around the election.

The following shows the network of users tweeting about the French election:

The following shows the network of hashtags related the French election:

Of course, once the graphs have settled into a local equilibrium, the exact positions of the nodes doesn't have an exact interpretation but broadly speaking the related hashtags cluster together. Most times the Fillon-related hashtags cluster together, then hashtags related to the debates cluster together, etc.


text:
<?php
namespace MyString\Controller\Plugin;

use Zend\Mvc\Controller\Plugin\AbstractPlugin;

class AddPrefixPlugin extends AbstractPlugin
{
    /**
     * @var int The number of 0's to prepend
     */
    private $strPrefix;

    public function __construct($strPrefix)
    {
        $this->strPrefix = $strPrefix;
    }

    /**
     * __invoke
     *
     * @param int $number The number to prepend zeros in
     *
     * @return string
     */
    public function __invoke($cadena)
    {

        while (strlen($cadena)<$this->strPrefix){
             $cadena = '*'.$cadena;
        }
        return ($cadena);

    }
}


text:
var arrayProto = [];

function max(){
    return Math.max.apply(Math, arrayProto.map.call(arguments, parseFloat));
}

function min(){
    return Math.min.apply(Math, arrayProto.map.call(arguments, parseFloat));
}

function px(value){
    return value + 'px';
}

module.exports = {
    floor: Math.floor,
    random: Math.random,
    max: max,
    min: min,
    px: px
};