text:
/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 *    DatabaseLoader.java
 *    Copyright (C) 2004-2012 University of Waikato, Hamilton, New Zealand
 *
 */

package weka.core.converters;

import java.io.File;
import java.io.IOException;
import java.sql.DatabaseMetaData;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.StringTokenizer;
import java.util.Vector;

import weka.core.Attribute;
import weka.core.DenseInstance;
import weka.core.Environment;
import weka.core.EnvironmentHandler;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.Option;
import weka.core.OptionHandler;
import weka.core.OptionMetadata;
import weka.core.RevisionUtils;
import weka.core.SparseInstance;
import weka.core.Utils;
import weka.experiment.InstanceQuery;
import weka.gui.FilePropertyMetadata;

import javax.swing.JFileChooser;

/**
 *  Reads Instances from a Database. Can read a
 * database in batch or incremental mode.

 * In inremental mode MySQL and HSQLDB are supported.

 * For all other DBMS set a pseudoincremental mode is used:

 * In pseudo incremental mode the instances are read into main memory all at
 * once and then incrementally provided to the user.

 * For incremental loading the rows in the database table have to be ordered
 * uniquely.

 * The reason for this is that every time only a single row is fetched by
 * extending the user query by a LIMIT clause.

 * If this extension is impossible instances will be loaded pseudoincrementally.
 * To ensure that every row is fetched exaclty once, they have to ordered.

 * Therefore a (primary) key is necessary.This approach is chosen, instead of
 * using JDBC driver facilities, because the latter one differ betweeen
 * different drivers.

 * If you use the DatabaseSaver and save instances by generating automatically a
 * primary key (its name is defined in DtabaseUtils), this primary key will be
 * used for ordering but will not be part of the output. The user defined SQL
 * query to extract the instances should not contain LIMIT and ORDER BY clauses
 * (see -Q option).

 * In addition, for incremental loading, you can define in the DatabaseUtils
 * file how many distinct values a nominal attribute is allowed to have. If this
 * number is exceeded, the column will become a string attribute.

 * In batch mode no string attributes will be created.
 *

 *
 *
 *  Valid options are:
 *

 *
 *

 * -url <JDBC URL>
 *  The JDBC URL to connect to.
 *  (default: from DatabaseUtils.props file)
 *

 *
 *

 * -user <name>
 *  The user to connect with to the database.
 *  (default: none)
 *

 *
 *

 * -password <password>
 *  The password to connect with to the database.
 *  (default: none)
 *

 *
 *

 * -Q <query>
 *  SQL query of the form
 *   SELECT <list of columns>|* FROM <table> [WHERE]
 *  to execute.
 *  (default: Select * From Results0)
 *

 *
 *

 * -P <list of column names>
 *  List of column names uniquely defining a DB row
 *  (separated by ', ').
 *  Used for incremental loading.
 *  If not specified, the key will be determined automatically,
 *  if possible with the used JDBC driver.
 *  The auto ID column created by the DatabaseSaver won't be loaded.
 *

 *
 *

 * -I
 *  Sets incremental loading
 *

 *
 *
 *
 * @author Stefan Mutter (mutter@cs.waikato.ac.nz)
 * @version $Revision: 11828 $
 * @see Loader
 */
public class DatabaseLoader extends AbstractLoader implements BatchConverter,
  IncrementalConverter, DatabaseConverter, OptionHandler, EnvironmentHandler {

  /** for serialization */
  static final long serialVersionUID = -7936159015338318659L;

  /**
   * The header information that is retrieved in the beginning of incremental
   * loading
   */
  protected Instances m_structure;

  /**
   * Used in pseudoincremental mode. The whole dataset from which instances will
   * be read incrementally.
   */
  protected Instances m_datasetPseudoInc;

  /**
   * Set of instances that equals m_structure except that the auto_generated_id
   * column is not included as an attribute
   */
  protected Instances m_oldStructure;

  /** The database connection */
  protected DatabaseConnection m_DataBaseConnection;

  /**
   * The user defined query to load instances. (form: SELECT
   * *|&ltcolumn-list> FROM &lttable> [WHERE <condition>])
   */
  protected String m_query = "Select * from Results0";

  /**
   * Flag indicating that pseudo incremental mode is used (all instances load at
   * once into main memeory and then incrementally from main memory instead of
   * the database)
   */
  protected boolean m_pseudoIncremental;

  /**
   * If true it checks whether or not the table exists in the database before
   * loading depending on jdbc metadata information. Set flag to false if no
   * check is required or if jdbc metadata is not complete.
   */
  protected boolean m_checkForTable;

  /**
   * Limit when an attribute is treated as string attribute and not as a nominal
   * one because it has to many values.
   */
  protected int m_nominalToStringLimit;

  /**
   * The number of rows obtained by m_query, eg the size of the ResultSet to
   * load
   */
  protected int m_rowCount;

  /** Indicates how many rows has already been loaded incrementally */
  protected int m_counter;

  /**
   * Decides which SQL statement to limit the number of rows should be used.
   * DBMS dependent. Algorithm just tries several possibilities.
   */
  protected int m_choice;

  /** Flag indicating that incremental process wants to read first instance */
  protected boolean m_firstTime;

  /**
   * Flag indicating that incremental mode is chosen (for command line use only)
   */
  protected boolean m_inc;

  /**
   * Contains the name of the columns that uniquely define a row in the
   * ResultSet. Ensures a unique ordering of instances for indremental loading.
   */
  protected ArrayList<String> m_orderBy;

  /** Stores the index of a nominal value */
  protected Hashtable<String, Double>[] m_nominalIndexes;

  /** Stores the nominal value */
  protected ArrayList<String>[] m_nominalStrings;

  /**
   * Name of the primary key column that will allow unique ordering necessary
   * for incremental loading. The name is specified in the DatabaseUtils file.
   */
  protected String m_idColumn;

  /** the JDBC URL to use */
  protected String m_URL = null;

  /** the database user to use */
  protected String m_User = "";

  /** the database password to use */
  protected String m_Password = "";

  /** the keys for unique ordering */
  protected String m_Keys = "";

  /** the custom props file to use instead of default one. */
  protected File m_CustomPropsFile = new File("${user.home}");

  /** Determines whether sparse data is created */
  protected boolean m_CreateSparseData = false;

  /** Environment variables */
  protected transient Environment m_env;

  /**
   * Constructor
   *
   * @throws Exception if initialization fails
   */
  public DatabaseLoader() throws Exception {

    resetOptions();
  }

  /**
   * Returns a string describing this Loader
   *
   * @return a description of the Loader suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String globalInfo() {
    return "Reads Instances from a Database. "
      + "Can read a database in batch or incremental mode.\n"
      + "In inremental mode MySQL and HSQLDB are supported.\n"
      + "For all other DBMS set a pseudoincremental mode is used:\n"
      + "In pseudo incremental mode the instances are read into main memory all at once and then incrementally provided to the user.\n"
      + "For incremental loading the rows in the database table have to be ordered uniquely.\n"
      + "The reason for this is that every time only a single row is fetched by extending the user query by a LIMIT clause.\n"
      + "If this extension is impossible instances will be loaded pseudoincrementally. To ensure that every row is fetched exaclty once, they have to ordered.\n"
      + "Therefore a (primary) key is necessary.This approach is chosen, instead of using JDBC driver facilities, because the latter one differ betweeen different drivers.\n"
      + "If you use the DatabaseSaver and save instances by generating automatically a primary key (its name is defined in DtabaseUtils), this primary key will "
      + "be used for ordering but will not be part of the output. The user defined SQL query to extract the instances should not contain LIMIT and ORDER BY clauses (see -Q option).\n"
      + "In addition, for incremental loading,  you can define in the DatabaseUtils file how many distinct values a nominal attribute is allowed to have. If this number is exceeded, the column will become a string attribute.\n"
      + "In batch mode no string attributes will be created.";
  }

  /**
   * Set the environment variables to use.
   *
   * @param env the environment variables to use
   */
  @Override
  public void setEnvironment(Environment env) {
    m_env = env;
    try {
      // force a new connection and setting of all parameters
      // with environment variables resolved
      m_DataBaseConnection = newDatabaseConnection();
      setUrl(m_URL);
      setUser(m_User);
      setPassword(m_Password);
    } catch (Exception ex) {
      // we won't complain about it here...
    }
  }

  private void checkEnv() {
    if (m_env == null) {
      m_env = Environment.getSystemWide();
    }
  }

  /**
   * Initializes a new DatabaseConnection object, either default one or from
   * custom props file.
   *
   * @return the DatabaseConnection object
   * @see #m_CustomPropsFile
   */
  protected DatabaseConnection newDatabaseConnection() throws Exception {
    DatabaseConnection result;

    checkEnv();

    if (m_CustomPropsFile != null) {
      File pFile = new File(m_CustomPropsFile.getPath());
      String pPath = m_CustomPropsFile.getPath();
      try {
        pPath = m_env.substitute(pPath);
        pFile = new File(pPath);
      } catch (Exception ex) {
      }
      result = new DatabaseConnection(pFile);
    } else {
      result = new DatabaseConnection();
    }

    m_pseudoIncremental = false;
    m_checkForTable = true;
    String props = result.getProperties().getProperty("nominalToStringLimit");
    m_nominalToStringLimit = Integer.parseInt(props);
    m_idColumn = result.getProperties().getProperty("idColumn");
    if (result.getProperties().getProperty("checkForTable", "")
      .equalsIgnoreCase("FALSE")) {
      m_checkForTable = false;
    }

    return result;
  }

  /**
   * Resets the Loader to the settings in either the default DatabaseUtils.props
   * or any property file that the user has specified via setCustomPropsFile().
   */
  public void resetOptions() {
    resetStructure();
    try {
      if (m_DataBaseConnection != null && m_DataBaseConnection.isConnected()) {
        m_DataBaseConnection.disconnectFromDatabase();
      }
      m_DataBaseConnection = newDatabaseConnection();
    } catch (Exception ex) {
      printException(ex);
    }

    m_URL = m_DataBaseConnection.getDatabaseURL();
    if (m_URL == null) {
      m_URL = "none set!";
    }
    m_User = m_DataBaseConnection.getUsername();
    if (m_User == null) {
      m_User = "";
    }
    m_Password = m_DataBaseConnection.getPassword();
    if (m_Password == null) {
      m_Password = "";
    }
    m_orderBy = new ArrayList<String>();
  }

  /**
   * Resets the Loader ready to read a new data set using set options
   *
   * @throws Exception if an error occurs while disconnecting from the database
   */
  @Override
  public void reset() {

    resetStructure();
    try {
      if (m_DataBaseConnection != null && m_DataBaseConnection.isConnected()) {
        m_DataBaseConnection.disconnectFromDatabase();
      }
      m_DataBaseConnection = newDatabaseConnection();
    } catch (Exception ex) {
      printException(ex);
    }

    // don't lose previously set connection data!
    if (m_URL != null) {
      setUrl(m_URL);
    }

    if (m_User != null) {
      setUser(m_User);
    }

    if (m_Password != null) {
      setPassword(m_Password);
    }

    m_orderBy = new ArrayList<String>();
    // don't lose previously set key columns!
    if (m_Keys != null) {
      String k = m_Keys;
      try {
        k = m_env.substitute(k);
      } catch (Exception ex) {
      }
      setKeys(k);
    }

    m_inc = false;
  }

  /**
   * Resets the structure of instances
   */
  public void resetStructure() {

    m_structure = null;
    m_datasetPseudoInc = null;
    m_oldStructure = null;
    m_rowCount = 0;
    m_counter = 0;
    m_choice = 0;
    m_firstTime = true;
    setRetrieval(NONE);
  }

  /**
   * Sets the query to execute against the database
   *
   * @param q the query to execute
   */
  public void setQuery(String q) {
    q = q.replaceAll("[fF][rR][oO][mM]", "FROM");
    q = q.replaceFirst("[sS][eE][lL][eE][cC][tT]", "SELECT");
    m_query = q;
  }

  /**
   * Gets the query to execute against the database
   *
   * @return the query
   */
  @OptionMetadata(displayName = "Query", description = "The query to execute",
    displayOrder = 4)
  public String getQuery() {
    return m_query;
  }

  /**
   * the tip text for this property
   *
   * @return the tip text
   */
  public String queryTipText() {

    return "The query that should load the instances."
      + "\n The query has to be of the form SELECT <column-list>|* FROM
 [WHERE <conditions>]";
  }

  /**
   * Sets the key columns of a database table
   *
   * @param keys a String containing the key columns in a comma separated list.
   */
  public void setKeys(String keys) {

    m_Keys = keys;
    m_orderBy.clear();
    StringTokenizer st = new StringTokenizer(keys, ",");
    while (st.hasMoreTokens()) {
      String column = st.nextToken();
      column = column.replaceAll(" ", "");
      m_orderBy.add(column);
    }
  }

  /**
   * Gets the key columns' name
   *
   * @return name of the key columns'
   */
  @OptionMetadata(
    displayName = "Key columns",
    description = "Specific key columns to use if "
      + "a primary key cannot be automatically detected. Used in incremental loading.",
    displayOrder = 5)
  public
    String getKeys() {

    StringBuffer key = new StringBuffer();
    for (int i = 0; i < m_orderBy.size(); i++) {
      key.append(m_orderBy.get(i));
      if (i != m_orderBy.size() - 1) {
        key.append(", ");
      }
    }
    return key.toString();
  }

  /**
   * the tip text for this property
   *
   * @return the tip text
   */
  public String keysTipText() {

    return "For incremental loading a unique identiefer has to be specified."
      + "\nIf the query includes all columns of a table (SELECT *...) a primary key"
      + "\ncan be detected automatically depending on the JDBC driver. If that is not possible"
      + "\nspecify the key columns here in a comma separated list.";
  }

  /**
   * Sets the custom properties file to use.
   *
   * @param value the custom props file to load database parameters from, use
   *          null or directory to disable custom properties.
   */
  public void setCustomPropsFile(File value) {
    m_CustomPropsFile = value;
  }

  /**
   * Returns the custom properties file in use, if any.
   *
   * @return the custom props file, null if none used
   */
  @OptionMetadata(
    displayName = "DB config file",
    description = "The custom properties that the user can use to override the default ones.",
    displayOrder = 8)
  @FilePropertyMetadata(fileChooserDialogType = JFileChooser.OPEN_DIALOG,
    directoriesOnly = false)
  public
  File getCustomPropsFile() {
    return m_CustomPropsFile;
  }

  /**
   * The tip text for this property.
   *
   * @return the tip text
   */
  public String customPropsFileTipText() {
    return "The custom properties that the user can use to override the default ones.";
  }

  /**
   * Sets the database URL
   *
   * @param url string with the database URL
   */
  @Override
  public void setUrl(String url) {
    checkEnv();

    m_URL = url;
    String dbU = m_URL;
    try {
      dbU = m_env.substitute(dbU);
    } catch (Exception ex) {
    }

    m_DataBaseConnection.setDatabaseURL(dbU);
  }

  /**
   * Gets the URL
   *
   * @return the URL
   */
  @OptionMetadata(displayName = "Database URL",
    description = "The URL of the database", displayOrder = 1)
  @Override
  public String getUrl() {

    // return m_DataBaseConnection.getDatabaseURL();
    return m_URL;
  }

  /**
   * the tip text for this property
   *
   * @return the tip text
   */
  public String urlTipText() {

    return "The URL of the database";
  }

  /**
   * Sets the database user
   *
   * @param user the database user name
   */
  @Override
  public void setUser(String user) {
    checkEnv();

    m_User = user;
    String userCopy = user;
    try {
      userCopy = m_env.substitute(userCopy);
    } catch (Exception ex) {
    }
    m_DataBaseConnection.setUsername(userCopy);
  }

  /**
   * Gets the user name
   *
   * @return name of database user
   */
  @OptionMetadata(displayName = "Username",
    description = "The user name for the database", displayOrder = 2)
  @Override
  public String getUser() {

    // return m_DataBaseConnection.getUsername();
    return m_User;
  }

  /**
   * the tip text for this property
   *
   * @return the tip text
   */
  public String userTipText() {

    return "The user name for the database";
  }

  /**
   * Sets user password for the database
   *
   * @param password the password
   */
  @Override
  public void setPassword(String password) {
    checkEnv();

    m_Password = password;
    String passCopy = password;
    try {
      passCopy = m_env.substitute(passCopy);
    } catch (Exception ex) {
    }
    m_DataBaseConnection.setPassword(password);
  }

  /**
   * Returns the database password
   *
   * @return the database password
   */
  @OptionMetadata(displayName = "Password",
    description = "The database password", displayOrder = 3)
  public String getPassword() {
    // return m_DataBaseConnection.getPassword();
    return m_Password;
  }

  /**
   * the tip text for this property
   *
   * @return the tip text
   */
  public String passwordTipText() {

    return "The database password";
  }

  /**
   * Returns the tip text for this property
   *
   * @return tip text for this property suitable for displaying in the
   *         explorer/experimenter gui
   */
  public String sparseDataTipText() {
    return "Encode data as sparse instances.";
  }

  /**
   * Sets whether data should be encoded as sparse instances
   *
   * @param s true if data should be encoded as a set of sparse instances
   */
  public void setSparseData(boolean s) {
    m_CreateSparseData = s;
  }

  /**
   * Gets whether data is to be returned as a set of sparse instances
   *
   * @return true if data is to be encoded as sparse instances
   */
  @OptionMetadata(displayName = "Create sparse instances", description = "Return sparse "
    + "rather than normal instances", displayOrder = 6)
  public boolean getSparseData() {
    return m_CreateSparseData;
  }

  /**
   * Sets the database url, user and pw
   *
   * @param url the database url
   * @param userName the user name
   * @param password the password
   */
  public void setSource(String url, String userName, String password) {

    try {
      m_DataBaseConnection = newDatabaseConnection();
      setUrl(url);
      setUser(userName);
      setPassword(password);
    } catch (Exception ex) {
      printException(ex);
    }
  }

  /**
   * Sets the database url
   *
   * @param url the database url
   */
  public void setSource(String url) {

    try {
      m_DataBaseConnection = newDatabaseConnection();
      setUrl(url);
      m_User = m_DataBaseConnection.getUsername();
      m_Password = m_DataBaseConnection.getPassword();
    } catch (Exception ex) {
      printException(ex);
    }
  }

  /**
   * Sets the database url using the DatabaseUtils file
   *
   * @throws Exception if something goes wrong
   */
  public void setSource() throws Exception {

    m_DataBaseConnection = newDatabaseConnection();
    m_URL = m_DataBaseConnection.getDatabaseURL();
    m_User = m_DataBaseConnection.getUsername();
    m_Password = m_DataBaseConnection.getPassword();
  }

  /**
   * Opens a connection to the database
   */
  public void connectToDatabase() {

    try {
      if (!m_DataBaseConnection.isConnected()) {
        m_DataBaseConnection.connectToDatabase();
      }
    } catch (Exception ex) {
      printException(ex);
    }
  }

  /**
   * Returns the table name or all after the FROM clause of the user specified
   * query to retrieve instances.
   *
   * @param onlyTableName true if only the table name should be returned, false
   *          otherwise
   * @return the end of the query
   */
  private String endOfQuery(boolean onlyTableName) {
    String table;
    int beginIndex, endIndex;

    beginIndex = m_query.indexOf("FROM ") + 5;
    while (m_query.charAt(beginIndex) == ' ') {
      beginIndex++;
    }
    endIndex = m_query.indexOf(" ", beginIndex);
    if (endIndex != -1 && onlyTableName) {
      table = m_query.substring(beginIndex, endIndex);
    } else {
      table = m_query.substring(beginIndex);
    }
    if (m_DataBaseConnection.getUpperCase()) {
      table = table.toUpperCase();
    }
    return table;
  }

  /**
   * Checks for a unique key using the JDBC driver's method: getPrimaryKey(),
   * getBestRowIdentifier(). Depending on their implementation a key can be
   * detected. The key is needed to order the instances uniquely for an
   * inremental loading. If an existing key cannot be detected, use -P option.
   *
   * @throws Exception if database error occurs
   * @return true, if a key could have been detected, false otherwise
   */
  private boolean checkForKey() throws Exception {

    String query = m_query;

    query = query.replaceAll(" +", " ");
    // query has to use all columns
    if (!query.startsWith("SELECT *")) {
      return false;
    }
    m_orderBy.clear();
    if (!m_DataBaseConnection.isConnected()) {
      m_DataBaseConnection.connectToDatabase();
    }
    DatabaseMetaData dmd = m_DataBaseConnection.getMetaData();
    String table = endOfQuery(true);
    // System.out.println(table);
    // check for primary keys
    ResultSet rs = dmd.getPrimaryKeys(null, null, table);
    while (rs.next()) {
      m_orderBy.add(rs.getString(4));
    }
    rs.close();
    if (m_orderBy.size() != 0) {
      return true;
    }
    // check for unique keys
    rs =
      dmd.getBestRowIdentifier(null, null, table,
        DatabaseMetaData.bestRowSession, false);
    ResultSetMetaData rmd = rs.getMetaData();
    int help = 0;
    while (rs.next()) {
      m_orderBy.add(rs.getString(2));
      help++;
    }
    rs.close();
    if (help == rmd.getColumnCount()) {
      m_orderBy.clear();
    }
    if (m_orderBy.size() != 0) {
      return true;
    }

    return false;
  }

  /**
   * Converts string attribute into nominal ones for an instance read during
   * incremental loading
   *
   * @param rs The result set
   * @param i the index of the nominal attribute
   * @throws Exception exception if it cannot be converted
   */
  private void stringToNominal(ResultSet rs, int i) throws Exception {

    while (rs.next()) {
      String str = rs.getString(1);
      if (!rs.wasNull()) {
        Double index = m_nominalIndexes[i - 1].get(str);
        if (index == null) {
          index = new Double(m_nominalStrings[i - 1].size());
          m_nominalIndexes[i - 1].put(str, index);
          m_nominalStrings[i - 1].add(str);
        }
      }
    }
  }

  /**
   * Used in incremental loading. Modifies the SQL statement, so that only one
   * instance per time is tretieved and the instances are ordered uniquely.
   *
   * @param query the query to modify for incremental loading
   * @param offset sets which tuple out of the uniquely ordered ones should be
   *          returned
   * @param choice the kind of query that is suitable for the used DBMS
   * @return the modified query that returns only one result tuple.
   */
  private String limitQuery(String query, int offset, int choice) {

    String limitedQuery;
    StringBuffer order = new StringBuffer();
    String orderByString = "";

    if (m_orderBy.size() != 0) {
      order.append(" ORDER BY ");
      for (int i = 0; i < m_orderBy.size() - 1; i++) {
        if (m_DataBaseConnection.getUpperCase()) {
          order.append(m_orderBy.get(i).toUpperCase());
        } else {
          order.append(m_orderBy.get(i));
        }
        order.append(", ");
      }
      if (m_DataBaseConnection.getUpperCase()) {
        order.append(m_orderBy.get(m_orderBy.size() - 1).toUpperCase());
      } else {
        order.append(m_orderBy.get(m_orderBy.size() - 1));
      }
      orderByString = order.toString();
    }
    if (choice == 0) {
      limitedQuery =
        query.replaceFirst("SELECT", "SELECT LIMIT " + offset + " 1");
      limitedQuery = limitedQuery.concat(orderByString);
      return limitedQuery;
    }
    if (choice == 1) {
      limitedQuery = query.concat(orderByString + " LIMIT 1 OFFSET " + offset);
      return limitedQuery;
    }
    limitedQuery = query.concat(orderByString + " LIMIT " + offset + ", 1");
    // System.out.println(limitedQuery);
    return limitedQuery;
  }

  /**
   * Counts the number of rows that are loaded from the database
   *
   * @throws Exception if the number of rows cannot be calculated
   * @return the entire number of rows
   */
  private int getRowCount() throws Exception {

    String query = "SELECT COUNT(*) FROM " + endOfQuery(false);
    if (m_DataBaseConnection.execute(query) == false) {
      throw new Exception("Cannot count results tuples.");
    }
    ResultSet rs = m_DataBaseConnection.getResultSet();
    rs.next();
    int i = rs.getInt(1);
    rs.close();
    return i;
  }

  /**
   * Determines and returns (if possible) the structure (internally the header)
   * of the data set as an empty set of instances.
   *
   * @return the structure of the data set as an empty set of Instances
   * @throws IOException if an error occurs
   */
  @Override
  public Instances getStructure() throws IOException {

    if (m_DataBaseConnection == null) {
      throw new IOException("No source database has been specified");
    }

    connectToDatabase();
    pseudo: try {
      if (m_pseudoIncremental && m_structure == null) {
        if (getRetrieval() == BATCH) {
          throw new IOException(
            "Cannot mix getting instances in both incremental and batch modes");
        }
        setRetrieval(NONE);
        m_datasetPseudoInc = getDataSet();
        m_structure = new Instances(m_datasetPseudoInc, 0);
        setRetrieval(NONE);
        return m_structure;
      }
      if (m_structure == null) {
        if (m_checkForTable) {
          if (!m_DataBaseConnection.tableExists(endOfQuery(true))) {
            throw new IOException(
              "Table does not exist according to metadata from JDBC driver. "
                + "If you are convinced the table exists, set 'checkForTable' "
                + "to 'False' in your DatabaseUtils.props file and try again.");
          }
        }

        // finds out which SQL statement to use for the DBMS to limit the number
        // of resulting rows to one
        int choice = 0;
        boolean rightChoice = false;
        while (!rightChoice) {
          try {
            String limitQ = limitQuery(m_query, 0, choice);
            if (m_DataBaseConnection.execute(limitQ) == false) {
              throw new IOException("Query didn't produce results");
            }
            m_choice = choice;
            rightChoice = true;
          } catch (SQLException ex) {
            choice++;
            if (choice == 3) {
              System.out
                .println("Incremental loading not supported for that DBMS. Pseudoincremental mode is used if you use incremental loading.\nAll rows are loaded into memory once and retrieved incrementally from memory instead of from the database.");
              m_pseudoIncremental = true;
              break pseudo;
            }
          }
        }

        String end = endOfQuery(false);
        ResultSet rs = m_DataBaseConnection.getResultSet();

        ResultSetMetaData md = rs.getMetaData();
        // rs.close();
        int numAttributes = md.getColumnCount();
        int[] attributeTypes = new int[numAttributes];
        m_nominalIndexes = Utils.cast(new Hashtable[numAttributes]);
        m_nominalStrings = Utils.cast(new ArrayList[numAttributes]);
        for (int i = 1; i <= numAttributes; i++) {
          switch (m_DataBaseConnection.translateDBColumnType(md
            .getColumnTypeName(i))) {
          case DatabaseConnection.STRING:

            String columnName = md.getColumnLabel(i);
            if (m_DataBaseConnection.getUpperCase()) {
              columnName = columnName.toUpperCase();
            }

            m_nominalIndexes[i - 1] = new Hashtable<String, Double>();
            m_nominalStrings[i - 1] = new ArrayList<String>();

            // fast incomplete structure for batch mode - actual
            // structure is determined by InstanceQuery in getDataSet()
            if (getRetrieval() != INCREMENTAL) {
              attributeTypes[i - 1] = Attribute.STRING;
              break;
            }
            // System.err.println("String --> nominal");
            ResultSet rs1;

            String query =
              "SELECT COUNT(DISTINCT( " + columnName + " )) FROM " + end;
            if (m_DataBaseConnection.execute(query) == true) {
              rs1 = m_DataBaseConnection.getResultSet();
              rs1.next();
              int count = rs1.getInt(1);
              rs1.close();
              // if(count > m_nominalToStringLimit ||
              // m_DataBaseConnection.execute("SELECT DISTINCT ( "+columnName+" ) FROM "+
              // end) == false){
              if (count > m_nominalToStringLimit
                || m_DataBaseConnection.execute("SELECT DISTINCT ( "
                  + columnName + " ) FROM " + end + " ORDER BY " + columnName) == false) {
                attributeTypes[i - 1] = Attribute.STRING;
                break;
              }
              rs1 = m_DataBaseConnection.getResultSet();
            } else {
              // System.err.println("Count for nominal values cannot be calculated. Attribute "+columnName+" treated as String.");
              attributeTypes[i - 1] = Attribute.STRING;
              break;
            }
            attributeTypes[i - 1] = Attribute.NOMINAL;
            stringToNominal(rs1, i);
            rs1.close();
            break;
          case DatabaseConnection.TEXT:
            // System.err.println("boolean --> string");

            columnName = md.getColumnLabel(i);
            if (m_DataBaseConnection.getUpperCase()) {
              columnName = columnName.toUpperCase();
            }

            m_nominalIndexes[i - 1] = new Hashtable<String, Double>();
            m_nominalStrings[i - 1] = new ArrayList<String>();

            // fast incomplete structure for batch mode - actual
            // structure is determined by InstanceQuery in getDataSet()
            if (getRetrieval() != INCREMENTAL) {
              attributeTypes[i - 1] = Attribute.STRING;
              break;
            }

            query = "SELECT COUNT(DISTINCT( " + columnName + " )) FROM " + end;
            if (m_DataBaseConnection.execute(query) == true) {
              rs1 = m_DataBaseConnection.getResultSet();
              stringToNominal(rs1, i);
              rs1.close();
            }
            attributeTypes[i - 1] = Attribute.STRING;
            break;
          case DatabaseConnection.BOOL:
            // System.err.println("boolean --> nominal");
            attributeTypes[i - 1] = Attribute.NOMINAL;
            m_nominalIndexes[i - 1] = new Hashtable<String, Double>();
            m_nominalIndexes[i - 1].put("false", new Double(0));
            m_nominalIndexes[i - 1].put("true", new Double(1));
            m_nominalStrings[i - 1] = new ArrayList<String>();
            m_nominalStrings[i - 1].add("false");
            m_nominalStrings[i - 1].add("true");
            break;
          case DatabaseConnection.DOUBLE:
            // System.err.println("BigDecimal --> numeric");
            attributeTypes[i - 1] = Attribute.NUMERIC;
            break;
          case DatabaseConnection.BYTE:
            // System.err.println("byte --> numeric");
            attributeTypes[i - 1] = Attribute.NUMERIC;
            break;
          case DatabaseConnection.SHORT:
            // System.err.println("short --> numeric");
            attributeTypes[i - 1] = Attribute.NUMERIC;
            break;
          case DatabaseConnection.INTEGER:
            // System.err.println("int --> numeric");
            attributeTypes[i - 1] = Attribute.NUMERIC;
            break;
          case DatabaseConnection.LONG:
            // System.err.println("long --> numeric");
            attributeTypes[i - 1] = Attribute.NUMERIC;
            break;
          case DatabaseConnection.FLOAT:
            // System.err.println("float --> numeric");
            attributeTypes[i - 1] = Attribute.NUMERIC;
            break;
          case DatabaseConnection.DATE:
            attributeTypes[i - 1] = Attribute.DATE;
            break;
          case DatabaseConnection.TIME:
            attributeTypes[i - 1] = Attribute.DATE;
            break;
          default:
            // System.err.println("Unknown column type");
            attributeTypes[i - 1] = Attribute.STRING;
          }
        }
        ArrayList<Attribute> attribInfo = new ArrayList<Attribute>();
        for (int i = 0; i < numAttributes; i++) {
          /* Fix for databases that uppercase column names */
          // String attribName = attributeCaseFix(md.getColumnName(i + 1));
          String attribName = md.getColumnLabel(i + 1);
          switch (attributeTypes[i]) {
          case Attribute.NOMINAL:
            attribInfo.add(new Attribute(attribName, m_nominalStrings[i]));
            break;
          case Attribute.NUMERIC:
            attribInfo.add(new Attribute(attribName));
            break;
          case Attribute.STRING:
            Attribute att = new Attribute(attribName, (ArrayList<String>) null);
            for (int n = 0; n < m_nominalStrings[i].size(); n++) {
              att.addStringValue(m_nominalStrings[i].get(n));
            }
            attribInfo.add(att);
            break;
          case Attribute.DATE:
            attribInfo.add(new Attribute(attribName, (String) null));
            break;
          default:
            throw new IOException("Unknown attribute type");
          }
        }
        m_structure = new Instances(endOfQuery(true), attribInfo, 0);
        // get rid of m_idColumn
        if (m_DataBaseConnection.getUpperCase()) {
          m_idColumn = m_idColumn.toUpperCase();
        }
        // System.out.println(m_structure.attribute(0).name().equals(idColumn));
        if (m_structure.attribute(0).name().equals(m_idColumn)) {
          m_oldStructure = new Instances(m_structure, 0);
          m_oldStructure.deleteAttributeAt(0);
          // System.out.println(m_structure);
        } else {
          m_oldStructure = new Instances(m_structure, 0);
        }

        if (m_DataBaseConnection.getResultSet() != null) {
          rs.close();
        }
      } else {
        if (m_oldStructure == null) {
          m_oldStructure = new Instances(m_structure, 0);
        }
      }
      m_DataBaseConnection.disconnectFromDatabase();
    } catch (Exception ex) {
      ex.printStackTrace();
      printException(ex);
    }
    return m_oldStructure;

  }

  /**
   * Return the full data set in batch mode (header and all intances at once).
   *
   * @return the structure of the data set as an empty set of Instances
   * @throws IOException if there is no source or parsing fails
   */
  @Override
  public Instances getDataSet() throws IOException {

    if (m_DataBaseConnection == null) {
      throw new IOException("No source database has been specified");
    }
    if (getRetrieval() == INCREMENTAL) {
      throw new IOException(
        "Cannot mix getting Instances in both incremental and batch modes");
    }
    setRetrieval(BATCH);

    Instances result = null;
    checkEnv();
    try {
      InstanceQuery iq = new InstanceQuery();
      iq.initialize(m_CustomPropsFile);
      String realURL = m_URL;
      try {
        realURL = m_env.substitute(realURL);
      } catch (Exception ex) {
      }
      iq.setDatabaseURL(realURL);
      String realUser = m_User;
      try {
        realUser = m_env.substitute(realUser);
      } catch (Exception ex) {
      }
      iq.setUsername(realUser);
      String realPass = m_Password;
      try {
        realPass = m_env.substitute(realPass);
      } catch (Exception ex) {
      }
      iq.setPassword(realPass);
      String realQuery = m_query;
      try {
        realQuery = m_env.substitute(realQuery);
      } catch (Exception ex) {
      }
      iq.setQuery(realQuery);
      iq.setSparseData(m_CreateSparseData);

      result = iq.retrieveInstances();

      if (m_DataBaseConnection.getUpperCase()) {
        m_idColumn = m_idColumn.toUpperCase();
      }

      if (result.attribute(0).name().equals(m_idColumn)) {
        result.deleteAttributeAt(0);
      }

      m_structure = new Instances(result, 0);
      iq.disconnectFromDatabase();

    } catch (Exception ex) {
      printException(ex);
      StringBuffer text = new StringBuffer();
      if (m_query.equals("Select * from Results0")) {
        text.append("\n\nDatabaseLoader options:\n");
        Enumeration<Option> enumi = listOptions();

        while (enumi.hasMoreElements()) {
          Option option = enumi.nextElement();
          text.append(option.synopsis() + '\n');
          text.append(option.description() + '\n');
        }
        System.out.println(text);
      }
    }

    return result;
  }

  /**
   * Reads an instance from a database.
   *
   * @param rs the ReusltSet to load
   * @throws Exception if instance cannot be read
   * @return an instance read from the database
   */
  private Instance readInstance(ResultSet rs) throws Exception {

    ResultSetMetaData md = rs.getMetaData();
    int numAttributes = md.getColumnCount();
    double[] vals = new double[numAttributes];
    m_structure.delete();
    for (int i = 1; i <= numAttributes; i++) {
      switch (m_DataBaseConnection.translateDBColumnType(md
        .getColumnTypeName(i))) {
      case DatabaseConnection.STRING:
        String str = rs.getString(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          Double index = m_nominalIndexes[i - 1].get(str);
          if (index == null) {
            index =
              new Double(m_structure.attribute(i - 1).addStringValue(str));
          }
          vals[i - 1] = index.doubleValue();
        }
        break;
      case DatabaseConnection.TEXT:
        str = rs.getString(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          Double index = m_nominalIndexes[i - 1].get(str);
          if (index == null) {
            index =
              new Double(m_structure.attribute(i - 1).addStringValue(str));
          }
          vals[i - 1] = index.doubleValue();
        }
        break;
      case DatabaseConnection.BOOL:
        boolean boo = rs.getBoolean(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          vals[i - 1] = (boo ? 1.0 : 0.0);
        }
        break;
      case DatabaseConnection.DOUBLE:
        // BigDecimal bd = rs.getBigDecimal(i, 4);
        double dd = rs.getDouble(i);
        // Use the column precision instead of 4?
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          // newInst.setValue(i - 1, bd.doubleValue());
          vals[i - 1] = dd;
        }
        break;
      case DatabaseConnection.BYTE:
        byte by = rs.getByte(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          vals[i - 1] = by;
        }
        break;
      case DatabaseConnection.SHORT:
        short sh = rs.getShort(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          vals[i - 1] = sh;
        }
        break;
      case DatabaseConnection.INTEGER:
        int in = rs.getInt(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          vals[i - 1] = in;
        }
        break;
      case DatabaseConnection.LONG:
        long lo = rs.getLong(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          vals[i - 1] = lo;
        }
        break;
      case DatabaseConnection.FLOAT:
        float fl = rs.getFloat(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          vals[i - 1] = fl;
        }
        break;
      case DatabaseConnection.DATE:
        Date date = rs.getDate(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          // TODO: Do a value check here.
          vals[i - 1] = date.getTime();
        }
        break;
      case DatabaseConnection.TIME:
        Time time = rs.getTime(i);
        if (rs.wasNull()) {
          vals[i - 1] = Utils.missingValue();
        } else {
          // TODO: Do a value check here.
          vals[i - 1] = time.getTime();
        }
        break;
      default:
        vals[i - 1] = Utils.missingValue();
      }
    }
    Instance inst;
    if (m_CreateSparseData) {
      inst = new SparseInstance(1.0, vals);
    } else {
      inst = new DenseInstance(1.0, vals);
    }
    // get rid of m_idColumn
    if (m_DataBaseConnection.getUpperCase()) {
      m_idColumn = m_idColumn.toUpperCase();
    }
    if (m_structure.attribute(0).name().equals(m_idColumn)) {
      inst.deleteAttributeAt(0);
      m_oldStructure.add(inst);
      inst = m_oldStructure.instance(0);
      m_oldStructure.delete(0);
    } else {
      // instances is added to and deleted from the structure to get the true
      // nominal values instead of the index of the values.
      m_structure.add(inst);
      inst = m_structure.instance(0);
      m_structure.delete(0);
    }
    return inst;

  }

  /**
   * Read the data set incrementally---get the next instance in the data set or
   * returns null if there are no more instances to get. If the structure hasn't
   * yet been determined by a call to getStructure then method does so before
   * returning the next instance in the data set.
   *
   * @param structure the dataset header information, will get updated in case
   *          of string or relational attributes
   * @return the next instance in the data set as an Instance object or null if
   *         there are no more instances to be read
   * @throws IOException if there is an error during parsing
   */
  @Override
  public Instance getNextInstance(Instances structure) throws IOException {

    m_structure = structure;

    if (m_DataBaseConnection == null) {
      throw new IOException("No source database has been specified");
    }
    if (getRetrieval() == BATCH) {
      throw new IOException(
        "Cannot mix getting Instances in both incremental and batch modes");
    }
    // pseudoInremental: Load all instances into main memory in batch mode and
    // give them incrementally to user
    if (m_pseudoIncremental) {
      setRetrieval(INCREMENTAL);
      if (m_datasetPseudoInc.numInstances() > 0) {
        Instance current = m_datasetPseudoInc.instance(0);
        m_datasetPseudoInc.delete(0);
        return current;
      } else {
        resetStructure();
        return null;
      }
    }
    // real incremental mode. At the moment(version 1.0) only for MySQL and
    // HSQLDB (Postgres not tested, should work)
    setRetrieval(INCREMENTAL);
    try {
      if (!m_DataBaseConnection.isConnected()) {
        connectToDatabase();
      }
      // if no key columns specified by user, try to detect automatically
      if (m_firstTime && m_orderBy.size() == 0) {
        if (!checkForKey()) {
          throw new Exception(
            "A unique order cannot be detected automatically.\nYou have to use SELECT * in your query to enable this feature.\nMaybe JDBC driver is not able to detect key.\nDefine primary key in your database or use -P option (command line) or enter key columns in the GUI.");
        }
      }
      if (m_firstTime) {
        m_firstTime = false;
        m_rowCount = getRowCount();
      }
      // as long as not all rows has been loaded
      if (m_counter < m_rowCount) {
        if (m_DataBaseConnection.execute(limitQuery(m_query, m_counter,
          m_choice)) == false) {
          throw new Exception("Tuple could not be retrieved.");
        }
        m_counter++;
        ResultSet rs = m_DataBaseConnection.getResultSet();
        rs.next();
        Instance current = readInstance(rs);
        rs.close();
        return current;
      } else {
        m_DataBaseConnection.disconnectFromDatabase();
        resetStructure();
        return null;
      }
    } catch (Exception ex) {
      printException(ex);
    }
    return null;
  }

  /**
   * Gets the setting
   *
   * @return the current setting
   */
  @Override
  public String[] getOptions() {

    Vector<String> options = new Vector<String>();

    if ((getUrl() != null) && (getUrl().length() != 0)) {
      options.add("-url");
      options.add(getUrl());
    }

    if ((getUser() != null) && (getUser().length() != 0)) {
      options.add("-user");
      options.add(getUser());
    }

    if ((getPassword() != null) && (getPassword().length() != 0)) {
      options.add("-password");
      options.add(getPassword());
    }

    options.add("-Q");
    options.add(getQuery());

    StringBuffer text = new StringBuffer();
    for (int i = 0; i < m_orderBy.size(); i++) {
      if (i > 0) {
        text.append(", ");
      }
      text.append(m_orderBy.get(i));
    }

    if (text.length() > 0) {
      options.add("-P");
      options.add(text.toString());
    }

    if (m_inc) {
      options.add("-I");
    }

    if ((m_CustomPropsFile != null) && !m_CustomPropsFile.isDirectory()) {
      options.add("-custom-props");
      options.add(m_CustomPropsFile.toString());
    }

    return options.toArray(new String[options.size()]);
  }

  /**
   * Lists the available options
   *
   * @return an enumeration of the available options
   */
  @Override
  public Enumeration<Option> listOptions() {

    Vector<Option> newVector = new Vector<Option>();

    newVector.add(new Option("\tThe JDBC URL to connect to.\n"
      + "\t(default: from DatabaseUtils.props file)", "url", 1,
      "-url <JDBC URL>"));

    newVector.add(new Option("\tThe user to connect with to the database.\n"
      + "\t(default: none)", "user", 1, "-user <name>"));

    newVector
      .add(new Option("\tThe password to connect with to the database.\n"
        + "\t(default: none)", "password", 1, "-password <password>"));

    newVector.add(new Option("\tSQL query of the form\n"
      + "\t\tSELECT <list of columns>|* FROM
 [WHERE]\n"
      + "\tto execute.\n" + "\t(default: Select * From Results0)", "Q", 1,
      "-Q <query>"));

    newVector.add(new Option(
      "\tList of column names uniquely defining a DB row\n"
        + "\t(separated by ', ').\n" + "\tUsed for incremental loading.\n"
        + "\tIf not specified, the key will be determined automatically,\n"
        + "\tif possible with the used JDBC driver.\n"
        + "\tThe auto ID column created by the DatabaseSaver won't be loaded.",
      "P", 1, "-P <list of column names>"));

    newVector.add(new Option("\tSets incremental loading", "I", 0, "-I"));

    newVector.addElement(new Option(
      "\tReturn sparse rather than normal instances.", "S", 0, "-S"));

    newVector.add(new Option(
      "\tThe custom properties file to use instead of default ones,\n"
        + "\tcontaining the database parameters.\n" + "\t(default: none)",
      "custom-props", 1, "-custom-props <file>"));

    return newVector.elements();
  }

  /**
   * Sets the options.
   *
   *  Valid options are:
   *

   *
   *

   * -url <JDBC URL>
   *  The JDBC URL to connect to.
   *  (default: from DatabaseUtils.props file)
   *

   *
   *

   * -user <name>
   *  The user to connect with to the database.
   *  (default: none)
   *

   *
   *

   * -password <password>
   *  The password to connect with to the database.
   *  (default: none)
   *

   *
   *

   * -Q <query>
   *  SQL query of the form
   *   SELECT <list of columns>|* FROM <table> [WHERE]
   *  to execute.
   *  (default: Select * From Results0)
   *

   *
   *

   * -P <list of column names>
   *  List of column names uniquely defining a DB row
   *  (separated by ', ').
   *  Used for incremental loading.
   *  If not specified, the key will be determined automatically,
   *  if possible with the used JDBC driver.
   *  The auto ID column created by the DatabaseSaver won't be loaded.
   *

   *
   *

   * -I
   *  Sets incremental loading
   *

   *
   *
   *
   * @param options the options
   * @throws Exception if options cannot be set
   */
  @Override
  public void setOptions(String[] options) throws Exception {

    String optionString, keyString, tmpStr;

    optionString = Utils.getOption('Q', options);

    keyString = Utils.getOption('P', options);

    reset();

    tmpStr = Utils.getOption("url", options);
    if (tmpStr.length() != 0) {
      setUrl(tmpStr);
    }

    tmpStr = Utils.getOption("user", options);
    if (tmpStr.length() != 0) {
      setUser(tmpStr);
    }

    tmpStr = Utils.getOption("password", options);
    if (tmpStr.length() != 0) {
      setPassword(tmpStr);
    }

    if (optionString.length() != 0) {
      setQuery(optionString);
    }

    m_orderBy.clear();

    m_inc = Utils.getFlag('I', options);

    if (m_inc) {
      StringTokenizer st = new StringTokenizer(keyString, ",");
      while (st.hasMoreTokens()) {
        String column = st.nextToken();
        column = column.replaceAll(" ", "");
        m_orderBy.add(column);
      }
    }

    tmpStr = Utils.getOption("custom-props", options);
    if (tmpStr.length() == 0) {
      setCustomPropsFile(null);
    } else {
      setCustomPropsFile(new File(tmpStr));
    }
  }

  /**
   * Prints an exception
   *
   * @param ex the exception to print
   */
  private void printException(Exception ex) {

    System.out.println("\n--- Exception caught ---\n");
    while (ex != null) {
      System.out.println("Message:   " + ex.getMessage());
      if (ex instanceof SQLException) {
        System.out.println("SQLState:  " + ((SQLException) ex).getSQLState());
        System.out.println("ErrorCode: " + ((SQLException) ex).getErrorCode());
        ex = ((SQLException) ex).getNextException();
      } else {
        ex = null;
      }
      System.out.println("");
    }

  }

  /**
   * Returns the revision string.
   *
   * @return the revision
   */
  @Override
  public String getRevision() {
    return RevisionUtils.extract("$Revision: 11828 $");
  }

  /**
   * Main method.
   *
   * @param options the options
   */
  public static void main(String[] options) {

    DatabaseLoader atf;
    try {
      atf = new DatabaseLoader();
      atf.setOptions(options);
      atf.setSource(atf.getUrl(), atf.getUser(), atf.getPassword());
      if (!atf.m_inc) {
        System.out.println(atf.getDataSet());
      } else {
        Instances structure = atf.getStructure();
        System.out.println(structure);
        Instance temp;
        do {
          temp = atf.getNextInstance(structure);
          if (temp != null) {
            System.out.println(temp);
          }
        } while (temp != null);
      }
    } catch (Exception e) {
      e.printStackTrace();
      System.out.println("\n" + e.getMessage());
    }
  }
}


text:
<title>FFTW 3.0.1</title>
<meta http-equiv="Content-Type" content="text/html">
<meta name=description content="FFTW 3.0.1">
<meta name=generator content="makeinfo 4.2">
<link href="http://www.gnu.org/software/texinfo/" rel=generator-home>

Node:Planner Flags,
Next:Real-data DFTs,
Previous:Complex DFTs,
Up:Basic Interface

4.3.2 Planner Flags

All of the planner routines in FFTW accept an integer flags
argument, which is a bitwise OR (|) of zero or more of the flag
constants defined below.  These flags control the rigor (and time) of
the planning process, and can also impose (or lift) restrictions on the
type of transform algorithm that is employed.

Planning-rigor flags

FFTW_ESTIMATE specifies that, instead of actual measurements of
different algorithms, a simple heuristic is used to pick a (probably
sub-optimal) plan quickly.  With this flag, the input/output arrays are
not overwritten during planning.

FFTW_MEASURE tells FFTW to find an optimized plan by actually
computing several FFTs and measuring their execution time.
Depending on your machine, this can take some time (often a few
seconds).  FFTW_MEASURE is the default planning option.

FFTW_PATIENT is like FFTW_MEASURE, but considers a wider
range of algorithms and often produces a "more optimal" plan
(especially for large transforms), but at the expense of several times
longer planning time (especially for large transforms).

FFTW_EXHAUSTIVE is like FFTW_PATIENT, but considers an
even wider range of algorithms, including many that we think are
unlikely to be fast, to produce the most optimal plan but with a
substantially increased planning time.

Algorithm-restriction flags

FFTW_DESTROY_INPUT specifies that an out-of-place transform is
allowed to overwrite its input array with arbitrary data; this
can sometimes allow more efficient algorithms to be employed.

FFTW_PRESERVE_INPUT specifies that an out-of-place transform must
not change its input array.  This is ordinarily the
default, except for c2r and hc2r (i.e. complex-to-real)
transforms for which FFTW_DESTROY_INPUT is the default.  In the
latter cases, passing FFTW_PRESERVE_INPUT will attempt to use
algorithms that do not destroy the input, at the expense of worse
performance; for multi-dimensional c2r transforms, however, no
input-preserving algorithms are implemented and the planner will return
NULL if one is requested.

FFTW_UNALIGNED specifies that the algorithm may not
impose any unusual alignment requirements on the input/output arrays
(i.e. no SIMD may be used).  This flag is normally not necessary,
since the planner automatically detects misaligned arrays.  The only use
for this flag is if you want to use the guru interface to execute a
given plan on a different array that may not be aligned like the
original.  (Using fftw_malloc makes this flag unnecessary even
then.)

redpajama_set_name: RedPajamaGithub


text:
/**
 * 左侧菜单
 */
.ui-menu {
    position: relative;
    cursor: pointer;
}
div.ui-menu-children div.ui-menu {
    font-size: 14px;
}
div.ui-menu-children div.ui-menu-children div.ui-menu {
    font-size: 12px;
}
.ui-menu-hover {
    background: #FFF;
}
.ui-menu-icon {
    position: absolute;
    background: url(img/icon.png) 0px 0px no-repeat;
}
.ui-menu-icon-new {
    width: 26px;
    height: 11px;
    overflow: hidden;
    position: absolute;
    background: url(img/icon.png) -34px -221px no-repeat;
    display: inline-block;
    margin-left: 3px;
    vertical-align: middle;
    *display: inline;
    *zoom: 1;
}
.ui-menu-children {
    width: 172px;
    border-bottom: 3px solid #CECECE;
    border-left: 1px solid #9ABCD1;
    border-right: 1px solid #9ABCD1;
    border-top: 1px solid #9ABCD1;
    color: #333;
}
.ui-menu-children a {
    display: block;
    height: 100%;
    text-decoration: none;
    color: #333;
}
.ui-menu-children a {
    blr: expression(this.onFocus=this.blur());
    outline: none;
}
.ui-menu-children a: focus {
    -moz-outline-style: none;
}
.ui-menu-children .ui-menu {
    height: 40px;
    padding-left: 30px;
    line-height: 40px;
    border-bottom: 1px solid #9ABCD1;
    background: #EAF3F8;
}
.ui-menu-children .ui-menu-hover {
    background: #FFF;
}
.ui-menu-children .ui-menu .ui-menu-icon {
    top: 14px;
    left: 15px;
    width: 7px;
    height: 12px;
    background-position: -20px -35px;
    overflow: hidden;
}
.ui-menu-children .ui-menu .ui-menu-icon-new {
    margin-top: 15px;
    *margin-top: 0px;
}
.ui-menu-children .ui-menu-expanded {
    border-bottom: 1px solid #7E8B92;
}
.ui-menu-children .ui-menu-expanded .ui-menu-icon {
    top: 16px;
    left: 12px;
    width: 12px;
    height: 7px;
    background-position: 0px -47px;
}
.ui-menu-children .ui-menu-selected {
    color: #FFF;
    background: #56ABD8;
    font-weight: bold;
}
.ui-menu-children .ui-menu-selected a {
    color: #FFF;
}
.ui-menu-children .ui-menu-half-selected {
    color: #2581B1;
    font-weight: bold;
}
.ui-menu-children .ui-menu-half-selected a {
    color: #2581B1;
}
.ui-menu-children .ui-menu-children {
    border: 0px;
    border-top: 1px solid #B0C2CC;
}
.ui-menu-children .ui-menu-children .ui-menu {
    padding-left: 45px;
    height: 30px;
    line-height: 30px;
    background: #D7EBF6;
}
.ui-menu-children .ui-menu-children .ui-menu-hover {
    background: #FFF;
}
.ui-menu-children .ui-menu-children .ui-menu .ui-menu-icon {
    top: 9px;
    left: 23px;
    width: 14px;
    height: 14px;
    background-position: -48px -19px;
}
.ui-menu-children .ui-menu-children .ui-menu .ui-menu-icon-new {
    margin-top: 10px;
    *margin-top: 0px;
}
.ui-menu-children .ui-menu-children .ui-menu-expanded .ui-menu-icon {
    background-position: -62px -19px;
}
.ui-menu-children .ui-menu-children .ui-menu-selected {
    color: #FFF;
    background: #56ABD8;
    font-weight: bold;
}
.ui-menu-children .ui-menu-children .ui-menu-selected a {
    color: #FFF;
}
.ui-menu-children .ui-menu-children .ui-menu-half-selected {
    color: #333;
}
.ui-menu-children .ui-menu-children .ui-menu-half-selected a {
    color: #333;
}
.ui-menu-children .ui-menu-children .ui-menu-children .ui-menu {
    height: 27px;
    line-height: 27px;
    color: #515050;
    background: #C3D1DA;
    border: 1px solid #B2BFC7;
    border-top: 0px;
    padding-left: 44px;
}
.ui-menu-children .ui-menu-children .ui-menu-children .ui-menu .ui-menu-icon-new {
    margin-top: 8px;
    *margin-top: 0px;
}
.ui-menu-children .ui-menu-children .ui-menu-children .ui-meznu-hover {
    background: #FFF;
}
.ui-menu-children .ui-menu-children .ui-menu-children .ui-menu-selected {
    color: #FFF;
    background: #56ABD8;
    border-color: #4D99C1;
}

/**
 * float menu
 */
.ui-float-menu {
    position: relative;
    cursor: default;
}
.ui-float-menu-pointer {
    cursor: pointer;
}
div.ui-float-menu-children div.ui-float-menu {
    font-size: 14px;
}
div.ui-float-menu-children div.ui-float-menu-children div.ui-float-menu {
    font-size: 12px;
}
.ui-float-menu-hover {
    background: #FFF;
}
.ui-float-menu-icon {
    position: absolute;
    background: url(img/icon.png) 0px 0px no-repeat;
}
.ui-float-menu-icon-arror {
    background: url(img/icon.png) -70px -385px no-repeat;
    width: 6px;
    height: 12px;
    overflow: hidden;
    margin: 0px 10px 0px 155px;
    left: 0px;
    top: 0px;
    display: block;
    position: absolute;
}
.ui-float-menu-icon-new {
    width: 26px;
    height: 11px;
    overflow: hidden;
    position: absolute;
    background: url(img/icon.png) -34px -221px no-repeat;
    display: inline-block;
    margin-left: 3px;
    vertical-align: middle;
    *display: inline;
    *zoom: 1;
}
.ui-float-menu-children {
    width: 172px;
    border-bottom: 3px solid #CECECE;
    border-left: 1px solid #9ABCD1;
    border-right: 1px solid #9ABCD1;
    border-top: 1px solid #9ABCD1;
    color: #333;
}
.ui-float-menu-children a {
    display: block;
    height: 100%;
    text-decoration: none;
    color: #333;
}
.ui-float-menu-children a {
    blr: expression(this.onFocus=this.blur());
    outline: none;
}
.ui-float-menu-children a: focus {
    -moz-outline-style: none;
}
.ui-float-menu-children .ui-float-menu {
    height: 40px;
    padding-left: 30px;
    line-height: 40px;
    border-bottom: 1px solid #9ABCD1;
    background: #EAF3F8;
}
.ui-float-menu-children .ui-float-menu-hover {
    background: #FFF;
    z-index: 1000;
}
.ui-float-menu-children .ui-float-menu .ui-float-menu-icon {
    top: 14px;
    left: 15px;
    width: 7px;
    height: 12px;
    background-position: -20px -35px;
    overflow: hidden;
}
.ui-float-menu-children .ui-float-menu .ui-float-menu-icon-new {
    margin-top: 15px;
    *margin-top: 0px;
}
.ui-float-menu-children .ui-float-menu-expanded {
    border-bottom: 1px solid #7E8B92;
}
.ui-float-menu-children .ui-float-menu-expanded .ui-float-menu-icon {
    top: 16px;
    left: 12px;
    width: 12px;
    height: 7px;
    background-position: 0px -47px;
}
.ui-float-menu-children .ui-float-menu-selected {
    color: #FFF;
    background: #56ABD8;
    font-weight: bold;
}
.ui-float-menu-children .ui-float-menu-selected a {
    color: #FFF;
}
.ui-float-menu-children .ui-float-menu-half-selected {
    color: #2581B1;
    font-weight: bold;
}
.ui-float-menu-children .ui-float-menu-half-selected a {
    color: #2581B1;
}
.ui-float-menu-children .ui-float-menu-icon-arror {
    margin-top: 15px;
}

.ui-float-menu-children .ui-float-menu-children {
    border: 0px;
    border-top: 1px solid #B0C2CC;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu {
    padding-left: 30px;
    height: 30px;
    line-height: 30px;
    background: #D7EBF6;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-hover {
    background: #FFF;
    z-index: 10000;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu .ui-float-menu-icon {
    top: 9px;
    left: 23px;
    width: 14px;
    height: 14px;
    background-position: -48px -19px;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu .ui-float-menu-icon-new {
    margin-top: 10px;
    *margin-top: 0px;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-expanded .ui-float-menu-icon {
    background-position: -62px -19px;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-selected {
    color: #FFF;
    background: #56ABD8;
    font-weight: bold;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-selected a {
    color: #FFF;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-half-selected {
    color: #333;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-half-selected a {
    color: #333;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-icon-arror {
    margin-top: 10px;
}

.ui-float-menu-children .ui-float-menu-children .ui-float-menu-children .ui-float-menu {
    height: 27px;
    line-height: 27px;
    color: #515050;
    background: #C3D1DA;
    border: 1px solid #B2BFC7;
    border-top: 0px;
    padding-left: 44px;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-children .ui-float-menu .ui-float-menu-icon-new {
    margin-top: 8px;
    *margin-top: 0px;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-children .ui-float-menu-hover {
    background: #FFF;
    z-index: 10000;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-children .ui-float-menu-selected {
    color: #FFF;
    background: #56ABD8;
    border-color: #4D99C1;
}
.ui-float-menu-children .ui-float-menu-children .ui-float-menu-children .ui-float-menu-icon-arror {
    margin-top: 8px;
}

.ui-float-menu-fake {
    position: absolute;
    border: solid rgb(63,133,180) 1px;
    box-shadow: 3px 3px 4px #999;
    -o-box-shadow: 3px 3px 4px #999;
    -moz-box-shadow: 3px 3px 4px #999;
    -webkit-box-shadow: 3px 3px 4px #999;
}
.ui-float-menu-floater {
    position: absolute;
    z-index: 100;
    *z-index: 10000;
    border: 1px solid #9ABCD1;
    padding: 10px 10px;
    background-color: white;
    box-shadow: 3px 3px 4px #999;
    -o-box-shadow: 3px 3px 4px #999;
    -moz-box-shadow: 3px 3px 4px #999;
    -webkit-box-shadow: 3px 3px 4px #999;

}
.ui-float-menu-floater .ui-float-menu-floater-line {

}
.ui-float-menu-floater .ui-float-menu-floater-line-separater {
    border-bottom: #E0E0E0 solid 1px;
    padding-bottom: 10px;
    margin-bottom: 10px;
}
.ui-float-menu-floater .ui-float-menu-floater-line .ui-float-menu-floater-radio-line-title {
    width: 70px;
    margin: 7px 10px 4px 2px;
    text-align: right;
    color: #2581B1;
    font-weight: bold;
    float: left;
}
.ui-float-menu-floater .ui-float-menu-floater-line .ui-float-menu-floater-radio-line-detail {
    width: 410px;
    float: left;
}


text:
<waltz-svelte-component component="$ctrl.AuthSourceOverview"
                        primary-entity-ref="$ctrl.parentEntityRef">
</waltz-svelte-component>

    <waltz-dynamic-section-navigation parent-entity-ref="$ctrl.parentEntityRef">
    </waltz-dynamic-section-navigation>

<waltz-dynamic-sections-view parent-entity-ref="$ctrl.parentEntityRef">
</waltz-dynamic-sections-view>


text:
<?php

namespace Tests\AcmePhp\Core\Challenge\Dns;

use AcmePhp\Core\Challenge\Dns\DnsDataExtractor;
use AcmePhp\Core\Challenge\Dns\DnsResolverInterface;
use AcmePhp\Core\Challenge\Dns\DnsValidator;
use AcmePhp\Core\Protocol\AuthorizationChallenge;

class DnsValidatorTest extends \PHPUnit_Framework_TestCase
{
    public function testSupports()
    {
        $typeDns = 'dns-01';
        $typeHttp = 'http-01';

        $mockExtractor = $this->prophesize(DnsDataExtractor::class);
        $stubChallenge = $this->prophesize(AuthorizationChallenge::class);

        $validator = new DnsValidator($mockExtractor->reveal());

        $stubChallenge->getType()->willReturn($typeDns);
        $this->assertTrue($validator->supports($stubChallenge->reveal()));

        $stubChallenge->getType()->willReturn($typeHttp);
        $this->assertFalse($validator->supports($stubChallenge->reveal()));
    }

    public function testIsValid()
    {
        $recordName = '_acme-challenge.bar.com.';
        $recordValue = 'record_value';

        $mockResolver = $this->prophesize(DnsResolverInterface::class);
        $mockResolver->getTxtEntries($recordName)->willReturn([$recordValue]);
        $mockExtractor = $this->prophesize(DnsDataExtractor::class);
        $stubChallenge = $this->prophesize(AuthorizationChallenge::class);

        $validator = new DnsValidator($mockExtractor->reveal(), $mockResolver->reveal());

        $mockExtractor->getRecordName($stubChallenge->reveal())->willReturn($recordName);
        $mockExtractor->getRecordValue($stubChallenge->reveal())->willReturn($recordValue);

        $this->assertTrue($validator->isValid($stubChallenge->reveal()));
    }

    public function testIsValidCheckRecordValue()
    {
        $recordName = '_acme-challenge.bar.com.';
        $recordValue = 'record_value';

        $mockResolver = $this->prophesize(DnsResolverInterface::class);
        $mockResolver->getTxtEntries($recordName)->willReturn(['somethingElse']);
        $mockExtractor = $this->prophesize(DnsDataExtractor::class);
        $stubChallenge = $this->prophesize(AuthorizationChallenge::class);

        $validator = new DnsValidator($mockExtractor->reveal(), $mockResolver->reveal());

        $mockExtractor->getRecordName($stubChallenge->reveal())->willReturn($recordName);
        $mockExtractor->getRecordValue($stubChallenge->reveal())->willReturn($recordValue);

        $this->assertFalse($validator->isValid($stubChallenge->reveal()));
    }
}


text:
This package supports automatic minification of JavaScript files that cause a specific behavior of TAJS.

(Currently only for Linux and Mac OS X, not Windows.)

## Setup

To use delta debugging, jsdelta (https://www.npmjs.com/package/jsdelta) needs to be installed:

```
cd extras/jsdelta
npm install
```
Alternatively, if you already have jsdelta installed elsewhere, set the location in tajs.properties:
```
jsdelta = /usr/bin/jsdelta/delta.js
```

It also needs the location of the Java VM, to run TAJS from jsdelta:
```
java = /usr/bin/java
```

In order to speed up the minification process, a server-system can be set up (see `DeltaMinimizerServer`).
It is run by `DeltaMinimizer` if one or more ports are specified in `tajs.properties`:
```
jsdeltaserverports = 9900 9901 9902 9903 9904
```

## Usage

The minification can be done by using the `dk.brics.tajs.jsdelta.DeltaMinimizer` class.
See the javadoc of that class.

A simple usage example:

```java
DeltaMinimizer.reduce(Paths.get("/path/to/some-file.js"), MyRunPredicate.class);
```

This will produce a minimized version of `some-file.js`. The minimized version satisfies the test method of `MyRunPredicate class`.
Note that there are no restrictions on how `MyRunPredicate` is implemented; it could in principle invoke another program than TAJS.

Sample output for the minification process can be seen below. Notice that the final version is both present as a file and as printed output.

```
Initializing and sanity-checking minimization...
Starting minimization...
Started JSDelta process for target: /tmp/test.js.dir430022395006916871/test.js (at time: 2017-06-13T14:11:47.064, in thread: main)
jsdelta (main):::	 (***expected behaviour***)
jsdelta (main):::	 Starting iteration #0
jsdelta (main):::	 Target: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_1.js (1 bytes)
...
...
...
jsdelta (main):::	 Transforming candidate /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_11.js
jsdelta (main):::	 Transforming candidate /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_13.js
jsdelta (main):::	 Starting iteration #1
jsdelta (main):::	 Target: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_15.js (1 bytes)
jsdelta (main):::	 Target: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_16.js (6 bytes)
jsdelta (main):::	 Target: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_17.js (48 bytes)
jsdelta (main):::	 Target: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_18.js (48 bytes)
jsdelta (main):::	 Transforming candidate /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_19.js
jsdelta (main):::	 Transforming candidate /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_21.js
jsdelta (main):::	 Minimisation finished; final version is at /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_smallest.js (77 bytes)
Ended JSDelta process for target: /tmp/test.js.dir430022395006916871/test.js, with result: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_smallest.js (at time: 2017-06-13T14:12:10.444, in thread: main)
Minimization finished:
	location: /tmp/jsdelta-21505qRRNPFHCxUsr/delta_js_smallest.js
	lines: 4
	characters: 76
	content:
`` `
(function () {
    var arg0 = arguments[0];
    var arg1 = arguments[1];
});
`` `
```

Note: the interaction between jsdelta and Java relies on some classpath hacking and repeated restarts of JVMs.
As such, users of this package should be careful not to rely on special classpath setups and non-persistent data between runs.

## Example minification predicates

See the classes in the `src/dk/brics/jsdelta/predicates/examples` directory for concrete examples of minifier predicates.
They can be adjusted easily by changing the path to the JavaScript file inside them.

(Unless entirely new examples are created, new classes should not be committed to the examples directory.)


text:
title:  Миссионерская история
date:   02/04/2021
---

####  Нарушение слуха не помеха в благовестии

Моя мать, адвентистка седьмого дня, рассказывала мне, что я, будучи младенцем, не откликался на ее голос, когда она звала меня.

Мама обратилась за помощью, когда мне было около двух лет. Она отправила меня в школу для людей с ограниченными возможностями, где меня обучали языку жестов, а также учили говорить. Я прижимал руку к горлу учительницы, когда она говорила, а затем пытался сам воспроизвести этот звук.

Моя мама плакала, потому что я не слышал ее голоса. «Не волнуйтесь, — говорила учительница. — Все будет хорошо».

Два часа в день я учился в школе для людей с ограниченными возможностями. Затем два часа — в адвентистской школе, в которой меня научили читать и писать, и, что самое главное, там я узнал о Боге.

Я ходил в адвентистскую школу до восьми лет. К сожалению, в ней не было учителей, знающих язык жестов, поэтому мама в итоге отправила меня в государственную школу, где были учителя, которые могли общаться со мной.

Впервые я встретил других ребят адвентистов с нарушениями слуха на организованной церковью конференции в Адвентистском университете Линда Виста. Я был очень рад пообщаться с другими молодыми людьми с такими же потребностями, как у меня. Подобные конференции проходили в течение следующих двух лет. Затем Южно-Американский дивизион организовал свою первую конференцию для лиц с особыми потребностями и провел это мероприятие в Университете Монтеморелос в Монтеррее, Мексика. На конференции во мне пробудилось особое желание служить

Богу в качестве пастора. Но как? Я не мог позволить себе оплатить обучение.

В конце конференции президент университета Исмаэль Кастильо неожиданно задал вопрос: «Кто-нибудь из вас хочет здесь учиться?»

Он предложил полную стипендию на обучение. Тогда я понял, что Бог призывает меня стать пастором, и встал.

Вот так я стал первым глухим студентом-богословом в университете Монтеморелос. Я уже учусь на втором курсе. Это сложно, потому что никто не знает языка жестов. Мне приходится читать по губам учителей. У меня не все получается так, как мне хотелось бы. Я провалил несколько экзаменов за первый курс, и теперь мне нужно их пересдать.

Я провел в Мехико несколько евангельских программ для людей с нарушениями слуха. Меня приглашают проповедовать в церкви, где есть глухие члены церкви.

В моем сердце есть огромное желание получить высшее образование и служить пастором. Я мечтаю поехать в Испанию в качестве миссионера для слабослышащих.

Пожалуйста, молитесь за глухих и слабослышащих. У всех нас есть мечты. Мы готовы делать великие дела для Господа.

В этом квартале пожертвования Тринадцатой субботы будут направлены на нужды центра влияния «Лучшая жизнь» для подготовки студентов к миссионерскому служению при Колумбийском адвентистском университете.


text:
<?xml version="1.0" ?>
<ontology>
	<concept key="diag">
		<name>Diagnoses (empty)</name>
		<tooltip>A concept for collection of diagnoses</tooltip>
		<narrower/><!-- narrower without concepts indicates an empty folder -->
	</concept>
	<concept key="example">
		<name>Example</name>
		<tooltip>Another example concept</tooltip>
		<narrower>
			<concept key="examplesub" code="LOINC:12345-6">
				<name>Integer sub concept</name>
				<tooltip>Example for a sub concept of numeric integer type</tooltip>
				<constraints>
					<datatype>Integer</datatype>
					<unit>Count</unit>
				</constraints>
				<!-- no narrower element indicates leaf -->
			</concept>
			<concept key="examplesub2">
				<name>String sub concept</name>
				<tooltip>Example for a sub concept of string type</tooltip>
				<constraints>
					<datatype>String</datatype>
				</constraints>
				<!-- no narrower element indicates leaf -->
			</concept>
			<concept key="etc" patient-count="42">
				<name>Etcetera...</name>
				<tooltip>Et cetera (with patient count)</tooltip>
			</concept>
			<concept key="/example/enum">
				<name>Enum sub concept</name>
				<constraints>
					<datatype>Enum</datatype>
					<enum>
						<value label="First">1.</value>
						<value>2.</value>
						<value>3.</value>
					</enum>
				</constraints>
			</concept>
		</narrower>
	</concept>
</ontology>


text:
namespace realm {
namespace util {

/// Get a description of the current system platform.
///
/// Returns a space-separated concatenation of `osname`, `sysname`, `release`,
/// `version`, and `machine` as returned by get_platform_info(PlatformInfo&).
std::string get_platform_info();

struct PlatformInfo {
    std::string osname;  ///< Equivalent to `uname -o` (Linux).
    std::string sysname; ///< Equivalent to `uname -s`.
    std::string release; ///< Equivalent to `uname -r`.
    std::string version; ///< Equivalent to `uname -v`.
    std::string machine; ///< Equivalent to `uname -m`.
};

/// Get a description of the current system platform.
void get_platform_info(PlatformInfo&);

// Implementation

inline std::string get_platform_info()
{
    PlatformInfo info;
    get_platform_info(info); // Throws
    return (info.osname + " " + info.sysname + " " + info.release + " " + info.version + " " +
            info.machine); // Throws
}

} // namespace util
} // namespace realm

#endif // REALM_UTIL_PLATFORM_INFO_HPP


text:
package us.mcsw.compserver.commands;

public class CommandQuit extends Command {

	public CommandQuit() {
		super("quit", "Quits the program", "quit - Quits the program");
	}

	@Override
	public boolean execute(String[] args) {
		System.exit(-1);
		return false;
	}

}


text:
""" A backup set orchestrates collections of backups """

import logging

logger = logging.getLogger("dumprestore")

class BackupDriver:

    def before_dump(self, archive):
        """ Check that everything is ok to back up to the specified archive. """

    def dump(self, archive):
        """ Perform the dump to the archive. """

    def after_dump(self, archive):
        """ Perform cleanup operations. """

    def before_restore(self, archive):
        """ Check that everything is ok before a restore """

    def restore(self, archive):
        """ Perform the restore """

    def after_restore(self, archive):
        """ Perform cleanup operations """

class BackupSet:

    """ Orchestrates a collection of backups. First performs operations on it's children, then uses it's driver if required """

    def __init__(self, name="master", driver=None):
        self.name = name
        self.__archive = None
        self.children = []
        self.parent = None
        self.driver = driver

    def addChild(self, child):
        child.parent = self
        self.children.append(child)

    def _get_archive(self):
        if self.__archive is None:
            return self.parent.archive.subarchive(self.name)
        else:
            return self.__archive

    def _set_archive(self, archive):
        self.__archive = archive

    archive = property(_get_archive, _set_archive)

    def before_dump(self):
        """ Perform pre-flight checks. Return True if they passed, or False if they failed. """
        logger.info("%s performing pre-dump checks" % self.name)
        checks = []
        for c in self.children:
            checks.append(c.before_dump())
        if self.driver is not None:
            checks.append(self.driver.before_dump(self.archive))
        return False not in checks

    def dump(self):
        """ Backup source data to temporary files, and return the names of the files. """
        for c in self.children:
            c.dump()
        if self.driver is not None:
            self.driver.dump(self.archive)

    def after_dump(self):
        """ Perform cleanup operations """
        for c in self.children:
            c.after_dump()
        if self.driver is not None:
            self.driver.after_dump(self.archive)

    def before_restore(self):
        """ Perform any checks required before restoring, return True on success, False on failure. """
        checks = []
        for c in self.children:
            checks.append(c.before_restore())
        if self.driver is not None:
            checks.append(self.driver.before_restore(self.archive))
        return False not in checks

    def restore(self):
        for c in self.children:
            c.restore()
        if self.driver is not None:
            self.driver.restore(self.archive)

    def after_restore(self):
        """ Perform cleanup operations """
        for c in self.children:
            c.after_restore()
        if self.driver is not None:
            self.driver.after_restore(self.archive)


text:
var origin = window.location.origin;
var pathname = window.location.pathname;
var baseUrl = origin + pathname.toLowerCase().replace("index-latest.html", "");

require.config({
    baseUrl: baseUrl,
    paths: {
        aurelia: baseUrl + "/scripts/aurelia",
        views: baseUrl + "/views",
        underscore: baseUrl + "/scripts/underscore/underscore.min",
        webcomponentsjs: baseUrl + "/scripts/webcomponentsjs"
    },
    shim: {
        underscore: {
            exports: "_"
        }
    }
});

require(["aurelia/aurelia-bundle-latest"], function (au) {
    require(["aurelia-bundle-manifest"], function (abm) {
        require(["aurelia-bootstrapper"], function (b) {
        });
    });
});


text:
import React from 'react';
import { Link } from 'react-router';

import requireAuth from './requireAuth';

const NavBar = () => {
  return (

      <Link to="/rooms" className="nav-bar__link" activeClassName="nav-bar__link--active">

      </Link>

  );
};

export default requireAuth(NavBar);


text:
class AccountsController < ApplicationController
  layout 'pages'

  # GET /settings
  # GET /settings.xml
  def show
    @account = Account.owner

    respond_to do |format|
      format.html # show.html.erb
      format.xml  { render :xml => @account }
    end
  end

  # PUT /settings
  # PUT /settings.xml
  def update
    @account = Account.owner

    respond_to do |format|
      if @account.update_attributes(params[:account])
        flash[:notice] = t('settings_updated')
        format.html { redirect_back_or_default(:action => 'show') }
        format.xml  { head :ok }
      else
        format.html { render :action => "show" }
        format.xml  { render :xml => @account.errors, :status => :unprocessable_entity }
      end
    end
  end

protected

  def authorized?(action = action_name, resource = nil)
    logged_in? and @logged_user.owner_of_owner?
  end

end


text:
/****************************************************************/
/* Parallel Combinatorial BLAS Library (for Graph Computations) */
/* version 1.4 -------------------------------------------------*/
/* date: 1/17/2014 ---------------------------------------------*/
/* authors: Aydin Buluc (abuluc@lbl.gov), Adam Lugowski --------*/
/****************************************************************/
/*
 Copyright (c) 2010-2014, The Regents of the University of California

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights
 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:

 The above copyright notice and this permission notice shall be included in
 all copies or substantial portions of the Software.

 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */

#ifndef _MEMORY_POOL_H
#define _MEMORY_POOL_H

#include <iostream>
#include <list>
#include <new>			// For "placement new" (classes using this memory pool may need it)
#include <fstream>

using namespace std;

//! EqualityComparable memory chunk object
//! Two memory chunks are considered equal if either their beginaddr or the endaddr are the same (just to facilitate deallocation)
class Memory
{
	public:
	Memory(char * m_beg, size_t m_size): begin(m_beg),size(m_size)
	{};

	char * begaddr() { return begin; }
	char * endaddr() { return begin + size; }

	bool operator < (const Memory & rhs) const
	{ return (begin < rhs.begin); }
	bool operator == (const Memory & rhs) const
	{ return (begin == rhs.begin); }

	char * begin;
	size_t size;
};

/**
  * \invariant Available memory addresses will be sorted w.r.t. their starting positions
  * \invariant At least one element exists in the freelist at any time.
  * \invariant Defragment on the fly: at any time, NO two consecutive chunks with chunk1.endaddr equals chunk2.begaddr exist
  */
class MemoryPool
{
public:
	MemoryPool(void * m_beg, size_t m_size);

	void * alloc(size_t size);
	void dealloc (void * base, size_t size);

	friend ofstream& operator<< (ofstream& outfile, const MemoryPool & mpool);

private:
	// std::list is a doubly linked list (i.e., a Sequence that supports both forward and backward traversal)
	list<Memory> freelist;
	char * initbeg;
	char * initend;
};

#endif


text:
<?php

/**
 * The cronjob keeps your software backuped, fast and clean. Set up the execution
 * intervals in the "app/config/Candy.inc.php" and lean back.
 *
 * @link http://github.com/marcoraddatz/candyCMS
 * @author Marco Raddatz <http://marcoraddatz.com>
 * @license MIT
 * @since 1.5
 *
 */

namespace CandyCMS\Plugins;

use CandyCMS\Core\Controllers\Logs;
use CandyCMS\Core\Controllers\Mails;
use CandyCMS\Core\Helpers\AdvancedException;
use CandyCMS\Core\Helpers\Helper;
use CandyCMS\Core\Helpers\I18n;
use CandyCMS\Core\Models\Main;
use PDO;

final class Cronjob {

  /**
   * The Id of the Cronjobs log entry, that is created for the execution
   *
   * @access private
   * @var int
   *
   */
  private $_iLogId = -1;

  /**
   * The user id of the user, that is running the Cronjob.
   *
   * @access private
   * @var int
   *
   */
  private $_iUserId;

  /**
   * Create a new Cronjob Object
   *
   * @access public
   * @param integer $iUserId user ID who updates the database.
   *
   */
  public function __construct($iUserId) {
    $this->_iUserId = $iUserId;

    $this->_startCronjob();
  }

  /**
   * Finish up the execution of the Cronjob
   *
   * @access public
   *
   */
  public function __destruct() {
    Logs::updateEndTime($this->_iLogId);
  }

  /**
   * start the execution of the Cronjob, so there wont be other simultanious executions
   *
   * @final
   * @access private
   *
   */
  private final function _startCronjob() {
    $oPDO = Main::$_oDbStatic;
    $oPDO->beginTransaction();

    # Create log entry, so other calls won't start aswell.
    $iTime = time();
    Logs::insert('cronjob', 'execute', 0, $this->_iUserId, $iTime, $iTime);

    # save the id, so we can update at end
    $this->_iLogId = Main::getLastInsertId();

    $oPDO->commit();
  }

  /**
   * Cleanup our temp folders.
   *
   * @final
   * @access public
   * @param array $aFolders temp folders to clean
   *
   */
  public final function cleanup($aFolders) {
    foreach ($aFolders as $sFolder) {
      $sTempPath = Helper::removeSlash(PATH_UPLOAD . '/temp/' . $sFolder);
      $oDir = opendir($sTempPath);

      while ($sFile = readdir($oDir)) {
        if (substr($sFile, 0, 1) == '.' || filemtime($sTempPath . '/' . $sFile) > strtotime("-10 days"))
          continue;

        unlink($sTempPath . '/' . $sFile);
      }
    }
  }

  /**
   * Optimize tables and delete old sessions.
   *
   * @final
   * @access public
   *
   */
  public final function optimize() {
    try {
      Main::$_oDbStatic->query("OPTIMIZE TABLE
                                  " . SQL_PREFIX . "blogs,
                                  " . SQL_PREFIX . "comments,
                                  " . SQL_PREFIX . "calendars,
                                  " . SQL_PREFIX . "contents,
                                  " . SQL_PREFIX . "downloads,
                                  " . SQL_PREFIX . "gallery_albums,
                                  " . SQL_PREFIX . "gallery_files,
                                  " . SQL_PREFIX . "migrations,
                                  " . SQL_PREFIX . "logs,
                                  " . SQL_PREFIX . "sessions,
                                  " . SQL_PREFIX . "users");
    }
    catch (AdvancedException $e) {
      Main::$_oDbStatic->rollBack();
      AdvancedException::reportBoth('0109 - ' . $e->getMessage());
      exit('SQL error.');
    }

    try {
      $oQuery = Main::$_oDbStatic->prepare("DELETE FROM
                                              " . SQL_PREFIX . "sessions
                                            WHERE
                                              date < :date");

      # Half a year
      $iDate = time() - 15552000;
      $oQuery->bindParam('date', $iDate, PDO::PARAM_INT);
      return $oQuery->execute();
    }
    catch (AdvancedException $e) {
      Main::$_oDbStatic->rollBack();
      AdvancedException::reportBoth('0109 - ' . $e->getMessage());
      exit('SQL error.');
    }
  }

  /**
   * Create a Backup of one Table
   *
   * @final
   * @access private
   * @param string $sTable the Table Name
   * @param string $sFileText the string to write the backup to
   * @return integer number of Columns
   * @throws AdvancedException
   *
   */
  private final function _backupTableInfo($sTable, &$sFileText) {
    $oQuery = Main::$_oDbStatic->query('SHOW COLUMNS FROM ' . $sTable);
    $aColumns = $oQuery->fetchAll(PDO::FETCH_ASSOC);
    $iColumns = count($aColumns);

    $sFileText .= <<<EOD
#---------------------------------------------------------------#
# Table: {$sTable}, Columns: {$iColumns}
#---------------------------------------------------------------#

EOD;

    $sFileText .= 'DROP TABLE IF EXISTS `' . $sTable . '`;';
    $sFileText .= "\r\n";
    $sFileText .= 'CREATE TABLE `' . $sTable . '` (';

    foreach ($aColumns as $aColumn) {
      $sFileText .= "\r\n`" . $aColumn['Field'] . '` ' . $aColumn['Type'];

      if (!empty($aColumn['Default']))
        $sFileText .= " NOT NULL default '" . $aColumn['Default'] . "'";

      elseif ($aColumn['Null'] !== 'YES')
        $sFileText .= ' NOT NULL';

      elseif ($aColumn['Null'] == 'YES')
        $sFileText .= ' default NULL';

      $sFileText .= ',';
    }
    $sFileText .= "\r\n";

    # Show extras like auto_increment etc
    try {
      $oQuery = Main::$_oDbStatic->query('SHOW KEYS FROM ' . $sTable);
      $aKeys = $oQuery->fetchAll(PDO::FETCH_ASSOC);

      $iKey = 1;
      foreach ($aKeys as $aKey) {
        $sKey = & $aKey['Key_name'];

        if (($sKey != 'PRIMARY') && ($sKey['Non_unique'] == 0))
          $sKey = 'UNIQUE|' . $sKey;

        # Do we have keys?
        if ($sKey == 'PRIMARY')
          $sFileText .= ' PRIMARY KEY (`' . $aKey['Column_name'] . '`)';

        elseif (substr($sKey, 0, 6) == "UNIQUE")
          $sFileText .= ' UNIQUE ' . substr($sKey, 7) . ' (`' . $aKey['Column_name'] . '`)';

        else
          $sFileText .= ' FULLTEXT KEY ' . $sKey . ' (`' . $aKey['Column_name'] . '`)';

        if (count($aKeys) !== $iKey)
          $sFileText .= ",\n";
        ++$iKey;
      }
    }
    catch (AdvancedException $e) {
      AdvancedException::reportBoth('0108 - ' . $e->getMessage());
      exit('SQL error.');
    }

    # Closing bracket
    $sFileText .= "\n)";

    try {
      # select last id
      $oQuery = Main::$_oDbStatic->query('SELECT
                                            id
                                          FROM
                                            ' . $sTable . '
                                          ORDER BY
                                            id DESC
                                          LIMIT
                                            1');

      $aRow = $oQuery->fetch(PDO::FETCH_ASSOC);
      $iRows = (int) $aRow['id'];
      $sFileText .= ' AUTO_INCREMENT=';
      $sFileText .= $iRows + 1;
    }
    catch (AdvancedException $e) {
      AdvancedException::reportBoth('0109 - ' . $e->getMessage());
      exit('SQL error.');
    }

    # We also use this as count for data entries
    $sFileText .= ' DEFAULT CHARSET=utf8;';
    $sFileText .= "\r\n\r\n";

    return $iColumns;
  }

  /**
   * Get the table data and write it to $sFileText
   *
   * @final
   * @access public
   * @param string $sTable the table to get the data from
   * @param string $sFileText the result string
   * @return int number of rows backed up
   * @throws AdvancedException
   *
   */
  private final function _backupTableData($sTable, &$sFileText, $iColumns) {
    # fetch content
    $oQuery = Main::$_oDbStatic->query('SELECT * FROM ' . $sTable);
    $aRows = $oQuery->fetchAll(PDO::FETCH_ASSOC);
    $iRows = count($aRows);

    $sFileText .= <<<EOD
#---------------------------------------------------------------#
# Data: {$sTable}, Rows: {$iRows}
#---------------------------------------------------------------#

EOD;

    foreach ($aRows as $aRow) {
      $sFileText .= 'INSERT INTO `' . $sTable . '` VALUES (';

      $iEntries = 1;
      foreach ($aRow as $sEntry) {
        $sFileText .= "'" . addslashes($sEntry) . "'";

        if ($iEntries !== $iColumns)
          $sFileText .= ',';

        $iEntries++;
      }

      $sFileText .= ");\r\n";
    }
    $sFileText .= "\r\n";
    return $iRows;
  }

  /**
   * Create a SQL backup.
   *
   * @final
   * @access public
   *
   */
  public final function backup() {
    $sBackupName      = date('Y-m-d_H-i');
    $sBackupFolder    = PATH_STANDARD . '/app/backup';
    $sBackupPath      = $sBackupFolder . '/' . $sBackupName . '.sql';

    Main::$_oDbStatic->beginTransaction();

    $sFileText = "#---------------------------------------------------------------#\r\n";
    $sFileText .= '# Server OS: '.@php_uname()."\r\n";
    $sFileText .= "#\r\n";
    $sFileText .= '# MySQL-Version: '.@mysql_get_server_info()."\r\n";
    $sFileText .= "#\r\n";
    $sFileText .= '# PHP-Version: '.@phpversion()."\r\n";
    $sFileText .= "#\r\n";
    $sFileText .= '# Database: ' . SQL_DB."\r\n";
    $sFileText .= "#\r\n";
    $sFileText .= "# Time of backup: ".date('Y-m-d H:i')."\r\n";
    $sFileText .= "#---------------------------------------------------------------#\r\n";
    $sFileText .= "\r\n";
    $sFileText .= "#---------------------------------------------------------------#\r\n";
    $sFileText .= "# Backup includes following tables:\r\n";
    $sFileText .= "#---------------------------------------------------------------#\r\n";

    # Get all tables and name them
    try {
      $oQuery = Main::$_oDbStatic->query("SHOW TABLES FROM " . SQL_DB . '_' . WEBSITE_MODE);
      $aResult = $oQuery->fetchAll();

      # Show all tables
      foreach ($aResult as $aTable) {
        $sFileText .= '# ' . $aTable[0] . "\r\n";
      }
      $sFileText .= "\r\n\r\n";

      # Now back them up
      foreach ($aResult as $aTable) {
        try {
          $iColumns = $this->_backupTableInfo($aTable[0], $sFileText);
          $this->_backupTableData($aTable[0], $sFileText, $iColumns);
        }
        catch (AdvancedException $e) {
          AdvancedException::reportBoth('0110 - ' . $e->getMessage());
          continue;
        }
      }
    }
    catch (AdvancedException $e) {
      AdvancedException::reportBoth('0111 - ' . $e->getMessage());
      exit('SQL error.');
    }

    # Write into file
    $oFile = @fopen($sBackupPath, 'a+');
    @fwrite($oFile, $sFileText);
    @fclose($oFile);

    if (PLUGIN_CRONJOB_GZIP_BACKUP === true) {
      $oData = implode('', file($sBackupPath));
      $oCompress = gzencode($oData, 9);
      unlink($sBackupPath);

      $sBackupPath = $sBackupPath . '.gz';
      $oF = fopen($sBackupPath, 'w+');
      fwrite($oF, $oCompress);
      fclose($oF);
    }

    # Send the backup via mail
    if (PLUGIN_CRONJOB_SEND_PER_MAIL === true) {
      $sMails = \CandyCMS\Core\Controllers\Main::__autoload('Mails');
      $sMails::send(WEBSITE_MAIL,
              I18n::get('cronjob.mail.subject', $sBackupName),
              I18n::get('cronjob.mail.body'),
              WEBSITE_MAIL_NOREPLY,
              $sBackupPath);
    }

    # Rollback, since we did only read statements
    Main::$_oDbStatic->rollBack();
  }

  /**
   * Return the status if we want to execute the cronjob.
   *
   * @static
   * @access public
   * @param integer $iInterval time in seconds that the cronjob should be executed
   * @return boolean update status
   *
   */
  public static function getNextUpdate($iInterval = '') {
    $iInterval = !empty($iInterval) ? $iInterval : PLUGIN_CRONJOB_UPDATE_INTERVAL;

    if (empty(Main::$_oDbStatic))
      Main::connectToDatabase();

    try {
      $oQuery = Main::$_oDbStatic->prepare("SELECT
                                              time_end
                                            FROM
                                              " . SQL_PREFIX . "logs
                                            WHERE
                                              controller_name = 'cronjob'
                                            ORDER BY
                                              time_end DESC
                                            LIMIT
                                              1");
      $oQuery->execute();
      $aResult = $oQuery->fetch(PDO::FETCH_ASSOC);

      return !$aResult ? true : (int)$aResult['time_end'] + $iInterval < time();
    }
    catch (AdvancedException $e) {
      AdvancedException::reportBoth('0108 - ' . $e->getMessage());
      exit('SQL error.');
    }
  }
}


text:
require 'spec_helper'

describe 'Users', :vcr do

  before do
    DesignerNews.reset!
    @client = DesignerNews::Client.new
  end

  context '.me' do
    it 'returns the User info as JSON' do
      @client.me
      assert_requested :get, designer_news_url("me")
    end
  end
end