<|start_filename|>src/org/simmetrics/metrics/JaroWinkler.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.commonPrefix;
import static java.lang.Math.min;

import org.simmetrics.StringDistance;
import org.simmetrics.StringMetric;

/**
 * Calculates the Jaro-Winkler distance (similarity) over two strings.
 * <p>
 * <code>
 * similarity(a,b) = jaro-winkler(a,b)
 * <br>
 * distance(a,b) = 1 - similarity(a,b)
 * </code>
 * <p>
 * Can be configured with a prefix adjustment scale, max prefix length and boost
 * threshold.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see <a
 *      href="http://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance">Wikipedia
 *      - Jaro-Winkler distance</a>
 * 
 * @see Jaro
 * 
 * 
 */
public final class JaroWinkler implements StringMetric, StringDistance {

	private final Jaro jaro = new Jaro();

	private static final float PREFIX_SCALE = 0.1f;
	private static final float WINKLER_BOOST_THRESHOLD = 0.7f;

	private static final float BOOST_THRESHOLD = 0.0f;
	private static final int MAX_PREFIX_LENGTH = 4;

	private final float boostThreshold;
	private final float prefixScale;
	private final int maxPrefixLength;

	/**
	 * Constructs a new JaroWinkler metric.
	 */
	public JaroWinkler() {
		this(BOOST_THRESHOLD, PREFIX_SCALE, MAX_PREFIX_LENGTH);
	}

	/**
	 * Constructs a new JaroWinkler metric with Winklers boost threshold of 0.7.
	 * 
	 * @return a new JaroWinkler metric
	 */
	public static JaroWinkler createWithBoostThreshold() {
		return new JaroWinkler(WINKLER_BOOST_THRESHOLD, PREFIX_SCALE,
				MAX_PREFIX_LENGTH);
	}

	/**
	 * Constructs a new JaroWinkler metric.
	 * 
	 * 
	 * @param boostThreshold
	 *            minimum jaro score for which the score is boosted
	 * @param prefixScale
	 *            scale at which a common prefix adds a bonus
	 * @param maxPrefixLength
	 *            cutoff at which a longer common prefix does not improve the
	 *            score
	 */
	public JaroWinkler(float boostThreshold, float prefixScale,
			int maxPrefixLength) {
		checkArgument(boostThreshold >= 0);
		checkArgument(0 <= prefixScale && prefixScale <= 1);
		checkArgument(maxPrefixLength >= 0);

		this.boostThreshold = boostThreshold;
		this.prefixScale = prefixScale;
		this.maxPrefixLength = maxPrefixLength;
	}

	@Override
	public float distance(String a, String b) {
		return 1.0f - compare(a, b);
	}
	
	@Override
	public float compare(final String a, final String b) {
		final float jaroScore = jaro.compare(a, b);

		if (jaroScore < boostThreshold) {
			return jaroScore;
		}

		int prefixLength = min(commonPrefix(a, b).length(), maxPrefixLength);

		return jaroScore + (prefixLength * prefixScale * (1.0f - jaroScore));
	}

	@Override
	public String toString() {
		return "JaroWinkler [boostThreshold=" + boostThreshold
				+ ", prefixScale=" + prefixScale + ", maxPrefixLength="
				+ maxPrefixLength + "]";
	}

}


<|start_filename|>src/utility/Chen.java<|end_filename|>
package utility;

import main.Column;
import main.Table;
import org.simmetrics.metrics.StringMetrics;

import java.util.*;

// Source: Fast Foreign-Key Detection in Microsoft SQL Server PowerPivot for Excel
public class Chen {
	private static final org.simmetrics.StringMetric metric = StringMetrics.jaroWinkler();
	private MultiSet<String> columnTokenCount;
	private MultiSet<String> tableTokenCount;

	public Chen(List<Table> tables) {
		columnTokenCount = getColumnTokenCounts(tables);
		tableTokenCount = getTableTokenCounts(tables);
	}

	public double getChenSimilarity(Column fk, Column pk) {
		List<Float> similarities = new ArrayList<>();   // Maximal similarities for each token in FK
		List<Double> frequencies = new ArrayList<>();    // Occurrence frequency for each token in FK

		for (String token1 : fk.getTokenizedLowerCaseTrimmedName()) {
			// Get the maximum similarity for token1
			// Note: we do not use Hungarian algorithm
			float similarityMax = -1;
			String bestMatch = "";
			for (String token2 : pk.getTokenizedLowerCaseTrimmedName()) {
				float similarity = metric.compare(token1, token2);
				if (similarity>similarityMax) {
					similarityMax = similarity;
					bestMatch = token2;
				}
			}
			similarities.add(similarityMax);

			// Get the occurrence frequency for the best matching token
			frequencies.add(1.0*columnTokenCount.count(bestMatch)/columnTokenCount.totalCount());
		}

		double nominator = 0.0;
		for (int i = 0; i < similarities.size(); i++) {
			nominator += (similarities.get(i) * Math.log(1.0 / frequencies.get(i)));
		}

		double denominator = 0.0;
		for (int i = 0; i < similarities.size(); i++) {
			denominator += Math.log(1.0 / frequencies.get(i));
		}

		return nominator / denominator;
	}

	public double getTableChenSimilarity(List<String> fkTokens, List<String> pkTokens) {
		List<Float> similarities = new ArrayList<>();   // Maximal similarities for each token in FK
		List<Double> frequencies = new ArrayList<>();    // Occurrence frequency for each token in FK

		for (String token1 : fkTokens) {
			// Get the maximum similarity for token1
			float similarity = 0;
			for (String token2 : pkTokens) {
				similarity = Math.max(similarity, metric.compare(token1, token2));
			}
			similarities.add(similarity);

			// Get the occurrence frequency for token1
			for (String token : fkTokens) {
				frequencies.add((tableTokenCount.count(token)+1.0)/(tableTokenCount.totalCount()+1.0)); // We always add 1 to avoid zeros in the denominator
			}
		}

		double nominator = 0.0;
		for (int i = 0; i < similarities.size(); i++) {
			nominator += (similarities.get(i) * Math.log(1.0 / frequencies.get(i)));
		}

		double denominator = 0.0;
		for (int i = 0; i < similarities.size(); i++) {
			denominator += Math.log(1.0 / frequencies.get(i));
		}

		return nominator / denominator;
	}

	// Pre-calculates the fraction of columns whose name contains token i.
	private MultiSet<String> getColumnTokenCounts(List<Table> tables) {
		MultiSet<String> multiSet = new MultiSet<>();
		Set<String> deduplicated = new HashSet<>();

		for (Table table : tables) {
			for (Column column : table.getColumnList()) {
				deduplicated.clear();
				deduplicated.addAll(column.getTokenizedLowerCaseTrimmedName());
				for (String token : deduplicated) {
					multiSet.add(token);
				}
			}
		}

		return multiSet;
	}

	// Pre-calculates the fraction of tables whose name contains token i.
	private MultiSet<String> getTableTokenCounts(List<Table> tables) {
		MultiSet<String> multiSet = new MultiSet<>();
		Set<String> deduplicated = new HashSet<>();

		for (Table table : tables) {
			deduplicated.clear();
			deduplicated.addAll(table.getTokenizedLowerCaseTrimmedName());
			for (String token : deduplicated) {
				multiSet.add(token);
			}
		}

		return multiSet;
	}

}


<|start_filename|>gui/controller/MainApp.java<|end_filename|>
package controller;

import javafx.application.Application;
import javafx.fxml.FXMLLoader;
import javafx.scene.Parent;
import javafx.scene.Scene;
import javafx.scene.image.Image;
import javafx.stage.Stage;

import java.io.IOException;

public class MainApp extends Application {

	// Main method is only for legacy support - Java 8 won't call it for a JavaFX application (it directly calls launch()).
	public static void main(String[] args) {
		launch(args);   // Tell Java to show the GUI
	}

	@Override
	// Define the GUI. Stage is OS window. Scene is the content of the window.
	// Note: All network connections are automatically closed on window closing - no need to handle it manually.
	public void start(Stage stage) throws IOException {
		Parent root = FXMLLoader.load(getClass().getResource("/fxml/main.fxml"));

		stage.setTitle("Linkifier");
		stage.getIcons().add(new Image("/img/network.png"));
		stage.setResizable(false);
		stage.setScene(new Scene(root));
		stage.show();
	}
}


<|start_filename|>src/utility/Levenshtein.java<|end_filename|>
package utility;

public class Levenshtein {

	// Source: https://rosettacode.org/wiki/Levenshtein_distance#Java
	// We avoid lower case transformation in this place because the
	// transformation, repeated million times, is time consuming -> use cached names.
	// Disadvantage: The result is a distance, not a similarity.
	// Speed: Jaro-Winkler is ~10 times faster than Levenshtein.
	// See: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4086698/pdf/1471-2105-15-187.pdf
	public static int getDistance(String a, String b) {
		int[] costs = new int[b.length() + 1];
		for (int j = 0; j < costs.length; j++)
			costs[j] = j;
		for (int i = 1; i <= a.length(); i++) {
			costs[0] = i;
			int nw = i - 1;
			for (int j = 1; j <= b.length(); j++) {
				int cj = Math.min(1 + Math.min(costs[j], costs[j - 1]), a.charAt(i - 1) == b.charAt(j - 1) ? nw : nw + 1);
				nw = costs[j];
				costs[j] = cj;
			}
		}
		return costs[b.length()];
	}
}


<|start_filename|>src/org/simmetrics/metrics/Levenshtein.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkArgument;
import static java.lang.Math.max;
import static org.simmetrics.metrics.Math.min;

import org.simmetrics.StringDistance;
import org.simmetrics.StringMetric;

/**
 * Calculates the Levenshtein distance and similarity over two strings.
 * <p>
 * Insert/delete and substitute operations can be weighted. When the cost for
 * substitution is zero Levenshtein does not satisfy the coincidence property.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see <a href=" http://en.wikipedia.org/wiki/Levenshtein_distance">Wikipedia -
 *      Levenshtein distance</a>
 * @see DamerauLevenshtein
 * 
 */
public final class Levenshtein implements StringMetric, StringDistance {

	private final float maxCost;
	private final float insertDelete;
	private final float substitute;

	/**
	 * Constructs a new weighted Levenshtein metric. When the cost for
	 * substitution is zero Levenshtein does not satisfy the coincidence
	 * property.
	 * 
	 * @param insertDelete
	 *            positive non-zero cost of an insert or deletion operation
	 * @param substitute
	 *            positive cost of a substitute operation
	 */
	public Levenshtein(float insertDelete, float substitute) {
		checkArgument(insertDelete > 0);
		checkArgument(substitute >= 0);
		this.maxCost = max(insertDelete, substitute);
		this.insertDelete = insertDelete;
		this.substitute = substitute;
	}

	/**
	 * Constructs a new Levenshtein metric.
	 */
	public Levenshtein() {
		this(1.0f, 1.0f);
	}

	@Override
	public float compare(final String a, final String b) {
		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		return 1.0f - (distance(a, b) / (maxCost * max(a.length(), b.length())));
	}

	@Override
	public float distance(final String s, final String t) {

		if (s.isEmpty())
			return t.length();
		if (t.isEmpty())
			return s.length();
		if (s.equals(t))
			return 0;

		final int tLength = t.length();
		final int sLength = s.length();

		float[] swap;
		float[] v0 = new float[tLength + 1];
		float[] v1 = new float[tLength + 1];

		// initialize v0 (the previous row of distances)
		// this row is A[0][i]: edit distance for an empty s
		// the distance is just the number of characters to delete from t
		for (int i = 0; i < v0.length; i++) {
			v0[i] = i * insertDelete;
		}

		for (int i = 0; i < sLength; i++) {

			// first element of v1 is A[i+1][0]
			// edit distance is delete (i+1) chars from s to match empty t
			v1[0] = (i + 1) * insertDelete;

			for (int j = 0; j < tLength; j++) {
				v1[j + 1] = min(v1[j] + insertDelete,
						v0[j + 1] + insertDelete,
						v0[j]
								+ (s.charAt(i) == t.charAt(j) ? 0.0f
										: substitute));
			}

			swap = v0;
			v0 = v1;
			v1 = swap;
		}

		// latest results was in v1 which was swapped with v0
		return v0[tLength];
	}

	@Override
	public String toString() {
		return "Levenshtein [insertDelete=" + insertDelete + ", substitute="
				+ substitute + "]";
	}

}


<|start_filename|>src/org/simmetrics/metrics/HammingDistance.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkArgument;

import java.util.Iterator;
import java.util.List;

import org.simmetrics.ListDistance;
import org.simmetrics.StringDistance;

import com.google.common.base.Objects;

/**
 * Calculates the Hamming distance distance between lists and strings.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see <a href="https://en.wikipedia.org/wiki/Hamming_distance">Wikipedia -
 *      Hamming distance</a>
 * 
 */
public final class HammingDistance {
	
	private HammingDistance() {
		//Don't construct
	}
	/**
	 * Hamming Distance algorithm to calculate distance between lists of equal
	 * size.
	 *
	 * <p>
	 * This class is immutable and thread-safe.
	 * 
	 * @param <T>
	 *            type of the token
	 * 
	 */
	private static final class HammingListDistance<T> implements
			ListDistance<T> {

		HammingListDistance() {
			// avoid synthetics
		}

		/**
		 * Measures the distance between lists {@code a} and {@code b} of equal
		 * size. The measurement results in a non-negative value. A value of
		 * {@code 0.0} indicates that {@code a} and {@code b} are similar.
		 * 
		 * @param a
		 *            list a to compare
		 * @param b
		 *            list b to compare
		 * @return a non-negative value
		 * @throws NullPointerException
		 *             when either a or b is null
		 * @throws IllegalArgumentException
		 *             when a and b differ in size
		 */
		@Override
		public float distance(List<T> a, List<T> b) {
			checkArgument(a.size() == b.size());

			if (a.isEmpty()) {
				return 0;
			}

			int distance = 0;

			Iterator<T> aItt = a.iterator();
			Iterator<T> bItt = b.iterator();

			while (aItt.hasNext()) {
				if (!Objects.equal(aItt.next(), bItt.next())) {
					distance++;
				}
			}

			return distance;
		}
		
		@Override
		public String toString() {
			return "HammingListDistance";
		}
	}

	/**
	 * Hamming Distance algorithm to calculate distance between strings of equal
	 * length.
	 *
	 * <p>
	 * This class is immutable and thread-safe.
	 * 
	 */
	private static final class HammingStringDistance implements StringDistance {

		HammingStringDistance() {
			// avoid synthetics
		}

		/**
		 * Measures the distance between strings {@code a} and {@code b} of
		 * equal length. The measurement results in a non-negative value. A
		 * value of {@code 0.0} indicates that {@code a} and {@code b} are
		 * similar.
		 * 
		 * @param a
		 *            string a to compare
		 * @param b
		 *            string b to compare
		 * @return a non-negative value
		 * @throws NullPointerException
		 *             when either a or b is null
		 * @throws IllegalArgumentException
		 *             when a and b differ in length
		 */
		@Override
		public float distance(String a, String b) {
			checkArgument(a.length() == b.length());

			if (a.isEmpty()) {
				return 0;
			}

			int distance = 0;
			for (int i = 0, length =  a.length(); i < length; i++) {
				if (a.charAt(i) != b.charAt(i)) {
					distance++;
				}
			}
			return distance;
		}
		
		@Override
		public String toString() {
			return "HammingStringDistance";
		}
	}

	/**
	 * Returns a Hamming distance to compare lists.
	 * 
	 * @return a Hamming distance to compare lists
	 */
	public static <T> ListDistance<T> forList() {
		return new HammingListDistance<>();
	}

	/**
	 * Returns a Hamming distance to compare strings.
	 * 
	 * @return a new Hamming distance to compare strings
	 */
	public static StringDistance forString() {
		return new HammingStringDistance();
	}
}


<|start_filename|>src/org/simmetrics/builders/StringMetrics.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

package org.simmetrics.builders;

import java.util.List;
import java.util.Set;

import org.simmetrics.Metric;
import org.simmetrics.StringMetric;
import org.simmetrics.simplifiers.Simplifier;
import org.simmetrics.tokenizers.Tokenizer;

import com.google.common.collect.Multiset;

@SuppressWarnings("deprecation") // Implementation of StringMetrics will migrate into this class
final class StringMetrics {

	public static StringMetric create(Metric<String> metric) {
		return org.simmetrics.metrics.StringMetrics.create(metric);
	}

	
	public static StringMetric create(Metric<String> metric, Simplifier simplifier) {
		return org.simmetrics.metrics.StringMetrics.create(metric,simplifier);
	}

	
	public static StringMetric createForListMetric(Metric<List<String>> metric, Simplifier simplifier,
			Tokenizer tokenizer) {
		return org.simmetrics.metrics.StringMetrics.createForListMetric(metric,simplifier, tokenizer);

	}

	
	public static StringMetric createForListMetric(Metric<List<String>> metric, Tokenizer tokenizer) {
		return org.simmetrics.metrics.StringMetrics.createForListMetric(metric, tokenizer);
	}

	
	public static StringMetric createForSetMetric(Metric<Set<String>> metric, Simplifier simplifier,
			Tokenizer tokenizer) {
		return org.simmetrics.metrics.StringMetrics.createForSetMetric(metric,simplifier, tokenizer);
	}

	
	public static StringMetric createForSetMetric(Metric<Set<String>> metric, Tokenizer tokenizer) {
		return org.simmetrics.metrics.StringMetrics.createForSetMetric(metric, tokenizer);
	}

	
	
	public static StringMetric createForMultisetMetric(Metric<Multiset<String>> metric, Simplifier simplifier,
			Tokenizer tokenizer) {
		return org.simmetrics.metrics.StringMetrics.createForMultisetMetric(metric,simplifier, tokenizer);
	}

	
	public static StringMetric createForMultisetMetric(Metric<Multiset<String>> metric, Tokenizer tokenizer) {
		return org.simmetrics.metrics.StringMetrics.createForMultisetMetric(metric, tokenizer);
	}


	private StringMetrics() {
		// Utility class.
	}

}


<|start_filename|>test/main/TableTest.java<|end_filename|>

package main;

import org.junit.Test;

import java.util.ArrayList;
import java.util.List;

import static org.junit.Assert.assertTrue;

public class TableTest {

	// Note: This method should be private or refactored if possible to NOT throw the exceptions.
	@Test (expected = IllegalStateException.class)
	public void toQuery() {
		Table table0 = new Table("schemaName", "tableName");
		table0.toQuery('`', '`');
	}

	@Test (expected = IllegalArgumentException.class)
	public void getHeader() {
		Table.getHeader(null);
	}

	@Test
    public void isJunctionTable2() throws Exception {
        List<Table> tables = new ArrayList<>();
        tables.add(new Table("schema", "employee"));
        tables.add(new Table("schema", "address"));
        tables.add(new Table("schema", "employee_address"));
        tables.add(new Table("schema", "irrelevant"));
        tables.add(new Table("schema", "phony_table"));
        tables.add(new Table("schema", "apple_banana_cherry"));
        tables.add(new Table("schema", "employee_copy"));
        tables.add(new Table("schema", "long_name"));
        tables.add(new Table("schema", "phony_long_name"));
        tables.add(new Table("schema", "employee_addressU"));
        tables.add(new Table("schema", "Amsterdam"));
        tables.add(new Table("schema", "Barcelona"));
        tables.add(new Table("schema", "AmsterdamBarcelona"));
        tables.add(new Table("schema", "employee2"));

        for (Table table : tables) {
            table.setLowerCaseTrimmedName(table.getName().toLowerCase());
        }

        assertTrue(tables.get(0).isJunctionTable2(tables) < 0.1);
        assertTrue(tables.get(1).isJunctionTable2(tables) < 0.1);
        assertTrue(tables.get(2).isJunctionTable2(tables) > 0.9);    // Two exactly matching tokens
        assertTrue(tables.get(3).isJunctionTable2(tables) < 0.1);
        assertTrue(tables.get(4).isJunctionTable2(tables) < 0.1);    // Two not matching tokens
        assertTrue(tables.get(5).isJunctionTable2(tables) < 0.1);    // Three not matching tokens
        assertTrue(tables.get(6).isJunctionTable2(tables) < 0.1);    // A copy
        assertTrue(tables.get(7).isJunctionTable2(tables) < 0.1);    // Two not matching tokens
        assertTrue(tables.get(8).isJunctionTable2(tables) < 0.1);
        assertTrue(tables.get(9).isJunctionTable2(tables) > 0.9);    // Small typo/plural form should be ignored
        assertTrue(tables.get(10).isJunctionTable2(tables) < 0.1);
        assertTrue(tables.get(11).isJunctionTable2(tables) < 0.1);
        assertTrue(tables.get(12).isJunctionTable2(tables) > 0.9);   // Just different separator
        assertTrue(tables.get(13).isJunctionTable2(tables) < 0.1);
    }
}


<|start_filename|>src/utility/TokenShareRatio.java<|end_filename|>
package utility;

import main.Column;
import main.Setting;

import java.util.Collections;
import java.util.List;

public class TokenShareRatio {

	// LCS works better than LD
	public static double setTokenShareRatioFkLD(List<String> tokensFk, List<String> tokensPk) {
		double shared = 0;
		double count = 0;

		for (String token : tokensPk) {
			if (tokensFk.contains(token) || Column.KEYWORDS.contains(token)) shared++;
			else if (Setting.SYNONYM_PERSON.contains(token) && !Collections.disjoint(Setting.SYNONYM_PERSON, tokensFk)) shared++;
			else if (Setting.SYNONYM_PRODUCT.contains(token) && !Collections.disjoint(Setting.SYNONYM_PRODUCT, tokensFk)) shared++;
			else {
				double maxSim = Integer.MIN_VALUE;
				for (String tokenFk : tokensFk) {
					double sim = 1.0*LCS.getLongestCommonSubsequence(token, tokenFk)/token.length();
					maxSim = Math.max(maxSim, sim);
				}
				shared += maxSim;
			}
			count++;
		}

		return shared / count;
	}

}


<|start_filename|>src/org/simmetrics/metrics/MongeElkan.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkArgument;
import static java.lang.Math.max;
import static java.lang.Math.sqrt;

import java.util.List;

import org.simmetrics.ListMetric;
import org.simmetrics.StringMetric;

/**
 * Calculates the normalized Monge-Elkan distance (similarity) over two strings.
 * The normalized Monge-Elkan distance is used because the the unnormalized
 * distance is not symmetric.
 * <p>
 * <code>
 * similarity(a,b) = sqrt(monge-elkan(a,b) * monge-elkan(b,a))
 * monge-elkan(a,b) = average( for s in a | max( for q in b | metric(s,q)) </code>
 * </code>
 * </p>
 * <p>
 * This class is immutable and thread-safe.
 * 
 */
public final class MongeElkan implements ListMetric<String> {

	private final StringMetric metric;

	/**
	 * Constructs a Monge-Elkan metric with metric.
	 * 
	 * @param metric
	 *            metric to use
	 */
	public MongeElkan(final StringMetric metric) {
		this.metric = metric;
	}

	@Override
	public float compare(List<String> a, List<String> b) {
		checkArgument(!a.contains(null), "a may not not contain null");
		checkArgument(!b.contains(null), "b may not not contain null");

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		// calculates normalized_similarity(a,b)
		return (float) sqrt(mongeElkan(a, b) * mongeElkan(b, a));
	}

	private float mongeElkan(List<String> a, List<String> b) {
		// calculates average( for s in a | max( for q in b | metric(s,q))
		float sum = 0.0f;

		for (String s : a) {
			float max = 0.0f;
			for (String q : b) {
				max = max(max, metric.compare(s, q));
			}
			sum += max;
		}
		return sum / a.size();
	}

	@Override
	public String toString() {
		return "MongeElkan [metric=" + metric + "]";
	}

}


<|start_filename|>src/org/simmetrics/metrics/StringMetrics.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkNotNull;
import static org.simmetrics.builders.StringMetricBuilder.with;
import static org.simmetrics.simplifiers.Simplifiers.chain;
import static org.simmetrics.tokenizers.Tokenizers.qGram;
import static org.simmetrics.tokenizers.Tokenizers.qGramWithPadding;
import static org.simmetrics.tokenizers.Tokenizers.whitespace;

import java.util.List;
import java.util.Set;

import org.simmetrics.Metric;
import org.simmetrics.StringMetric;
import org.simmetrics.builders.StringMetricBuilder;
import org.simmetrics.simplifiers.Simplifier;
import org.simmetrics.simplifiers.Soundex;
import org.simmetrics.tokenizers.Tokenizer;

import com.google.common.collect.Multiset;

/**
 * Utility class for string similarity metrics.
 * <p>
 * Consists of well known string similarity metrics and methods to create string
 * similarity metrics from list- or set metrics. All metrics are setup with
 * sensible defaults, to customize metrics use {@link StringMetricBuilder}.
 * <p>
 * The created similarity metrics are immutable and thread-safe provided all their
 * components are also immutable and thread-safe.
 */
public final class StringMetrics {

	/**
	 * Returns a cosine similarity metric over tokens in a string. The tokens
	 * are created by splitting the string on whitespace.
	 * 
	 * @return a cosine similarity metric
	 * 
	 * @see CosineSimilarity
	 */
	public static StringMetric cosineSimilarity() {
		return with(new CosineSimilarity<String>())
			.tokenize(whitespace())
			.build();
	}

	/**
	 * Returns a block distance similarity metric over tokens in a string. The
	 * tokens are created by splitting the string on whitespace.
	 * 
	 * @return a block distance metric
	 * 
	 * @see BlockDistance
	 */
	public static StringMetric blockDistance() {
		return with(new BlockDistance<String>()).tokenize(whitespace()).build();
	}

	/**
	 * Returns a Damerau-Levenshtein similarity metric over tokens in a string.
	 * The tokens are created by splitting the string on whitespace.
	 * 
	 * @return a Damerau-Levenshtein metric
	 * 
	 * @see DamerauLevenshtein
	 */
	public static StringMetric damerauLevenshtein() {
		return new DamerauLevenshtein();
	}

	/**
	 * Returns a Dice similarity metric over tokens in a string. The tokens are
	 * created by splitting the string on whitespace.
	 * 
	 * @return a dice metric
	 * 
	 * @see Dice
	 */
	public static StringMetric dice() {
		return with(new Dice<String>()).tokenize(whitespace()).build();
	}

	/**
	 * Returns an Euclidean distance similarity metric over tokens in a string.
	 * The tokens are created by splitting the string on whitespace.
	 * 
	 * @return a Euclidean distance similarity metric
	 * 
	 * @see EuclideanDistance
	 */
	public static StringMetric euclideanDistance() {
		return with(new EuclideanDistance<String>()).tokenize(whitespace())
				.build();
	}

	/**
	 * Returns a generalized Jaccard similarity metric over tokens in a string.
	 * The tokens are created by splitting the string on whitespace.
	 * 
	 * @return a generalized Jaccard index metric
	 * 
	 * @see GeneralizedJaccard
	 */
	public static StringMetric generalizedJaccard() {
		return with(new GeneralizedJaccard<String>()).tokenize(whitespace())
				.build();
	}

	/**
	 * Returns an identity string similarity metric. The metric returns 1.0 when
	 * the inputs are equals, and 0.0 when they're not.
	 * 
	 * @return an identity similarity metric
	 * 
	 * @see Identity
	 */
	public static StringMetric identity() {
		return new StringMetric() {

			private final Identity<String> metric = new Identity<>();

			@Override
			public float compare(String a, String b) {
				return metric.compare(a, b);
			}

			@Override
			public String toString() {
				return metric.toString();
			}
		};
	}

	/**
	 * Returns a Jaccard similarity metric over tokens in a string. The
	 * tokens are created by splitting the string on whitespace.
	 * 
	 * @return a Jaccard similarity metric
	 * 
	 * @see Jaccard
	 */
	public static StringMetric jaccard() {
		return with(new Jaccard<String>()).tokenize(whitespace()).build();
	}

	/**
	 * Returns a Jaro string similarity metric.
	 * 
	 * @return a Jaro string similarity metric
	 * 
	 * @see Jaro
	 */
	public static StringMetric jaro() {
		return new Jaro();
	}

	/**
	 * Returns a Jaro-Winkler string similarity metric.
	 * 
	 * @return a Jaro-Winkler string similarity metric
	 * 
	 * @see JaroWinkler
	 */
	public static StringMetric jaroWinkler() {
		return new JaroWinkler();
	}

	/**
	 * Returns a Levenshtein string similarity metric.
	 * 
	 * @return a Levenshtein string similarity metric
	 * 
	 * @see Levenshtein
	 */
	public static StringMetric levenshtein() {
		return new Levenshtein();
	}

	/**
	 * Returns a normalized Monge-Elkan metric over tokens in a string. The
	 * tokens are created by splitting the string on whitespace. The metric
	 * applies Smith-Waterman-Gotoh internally.
	 * 
	 * @return a normalized Monge-Elkan metric
	 * 
	 * @see MongeElkan
	 */
	public static StringMetric mongeElkan() {
		return with(new MongeElkan(new SmithWatermanGotoh())).tokenize(
				whitespace()).build();
	}

	/**
	 * Returns a Needleman-Wunch string similarity metric.
	 * 
	 * @return a Needleman-Wunch string similarity metric
	 * 
	 * @see NeedlemanWunch
	 */
	public static StringMetric needlemanWunch() {
		return new NeedlemanWunch();
	}

	/**
	 * Returns an overlap coefficient similarity metric over tokens in a string.
	 * The tokens are created by splitting the string on whitespace.
	 * 
	 * @return an overlap coefficient metric
	 * 
	 * @see OverlapCoefficient
	 */
	public static StringMetric overlapCoefficient() {
		return with(new OverlapCoefficient<String>()).tokenize(whitespace())
				.build();
	}

	/**
	 * Returns a q-grams distance similarity metric. Q-grams distance applies a
	 * block distance similarity similarity metric over all tri-grams in a
	 * string.
	 * 
	 * @return a q-grams distance similarity metric
	 * 
	 * @see BlockDistance
	 */
	public static StringMetric qGramsDistance() {
		return with(new BlockDistance<String>()).tokenize(qGramWithPadding(3))
				.build();
	}

	/**
	 * Returns a Simon White similarity metric. Simon White applies the
	 * quantitative version Dice similarity over tokens in a string. The tokens
	 * are created by splitting the string on whitespace and taking bi-grams of
	 * the created tokens.
	 * <p>
	 * Implementation based on the ideas as outlined in <a
	 * href="http://www.catalysoft.com/articles/StrikeAMatch.html">How to Strike
	 * a Match</a> by <cite><NAME></cite>.
	 * 
	 * @return a Simon White similarity metric
	 * 
	 * @see SimonWhite
	 */
	public static StringMetric simonWhite() {
		return with(new SimonWhite<String>()).tokenize(whitespace())
				.tokenize(qGram(2)).build();
	}

	/**
	 * Returns a Smith-Waterman string similarity metric.
	 * 
	 * @return a Smith-Waterman string similarity metric
	 * 
	 * @see SmithWaterman
	 */
	public static StringMetric smithWaterman() {
		return new SmithWaterman();
	}

	/**
	 * Returns a Smith-Waterman-Gotoh string similarity metric.
	 * 
	 * @return a Smith-Waterman-Gotoh string similarity metric
	 * 
	 * @see SmithWatermanGotoh
	 */
	public static StringMetric smithWatermanGotoh() {
		return new SmithWatermanGotoh();
	}

	/**
	 * Returns a soundex similarity metric. The metric applies the Jaro-Winkler
	 * similarity metric over soundex strings.
	 * 
	 * @return a soundex similarity metric
	 * 
	 * @see Soundex
	 * @see JaroWinkler
	 * 
	 * @deprecated will be removed due to a lack of a good use case
	 */
	@Deprecated
	public static StringMetric soundex() {
		return with(new JaroWinkler()).simplify(new Soundex()).build();
	}

	/**
	 * Returns a string similarity metric that uses the
	 * {@link LongestCommonSubsequence} metric.
	 * 
	 * @return a longest common subsequence metric
	 */
	public static StringMetric longestCommonSubsequence() {
		return new LongestCommonSubsequence();
	}

	/**
	 * Returns a similarity metric that uses the {@link LongestCommonSubstring}
	 * metric.
	 * 
	 * @return a longest common substring metric
	 */
	public static StringMetric longestCommonSubstring() {
		return new LongestCommonSubstring();
	}

	/**
	 * Either constructs a new string similarity metric or returns the original
	 * metric.
	 * 
	 * @param metric
	 *            a metric for strings
	 * 
	 * @return a similarity metric.
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric create(Metric<String> metric) {
		if (metric instanceof StringMetric) {
			return (StringMetric) metric;
		}

		return new ForString(metric);
	}

	/**
	 * Constructs a new composite string similarity metric. The simplifier will
	 * be applied before the metric compares the strings.
	 * 
	 * @param metric
	 *            a list metric
	 * @param simplifier
	 *            a simplifier
	 * @return a new composite similarity metric
	 * 
	 * @throws NullPointerException
	 *             when either metric or simplifier are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric create(Metric<String> metric,
			Simplifier simplifier) {
		if (metric instanceof ForString) {
			ForString forString = (ForString) metric;
			return new ForStringWithSimplifier(forString.getMetric(),
					simplifier);
		} else if (metric instanceof ForStringWithSimplifier) {
			ForStringWithSimplifier fsws = (ForStringWithSimplifier) metric;
			return new ForStringWithSimplifier(fsws.getMetric(), chain(
					simplifier, fsws.getSimplifier()));
		} else if (metric instanceof ForList) {
			ForList fl = (ForList) metric;
			return createForListMetric(fl.getMetric(), simplifier,
					fl.getTokenizer());
		} else if (metric instanceof ForListWithSimplifier) {
			ForListWithSimplifier fl = (ForListWithSimplifier) metric;
			return createForListMetric(fl.getMetric(),
					chain(simplifier, fl.getSimplifier()), fl.getTokenizer());
		} else if (metric instanceof ForSet) {
			ForSet fl = (ForSet) metric;
			return createForSetMetric(fl.getMetric(), simplifier,
					fl.getTokenizer());
		} else if (metric instanceof ForSetWithSimplifier) {
			ForSetWithSimplifier fl = (ForSetWithSimplifier) metric;
			return createForSetMetric(fl.getMetric(),
					chain(simplifier, fl.getSimplifier()), fl.getTokenizer());
		}

		return new ForStringWithSimplifier(metric, simplifier);
	}

	/**
	 * Creates a new composite string similarity metric.The tokenizer is used to
	 * tokenize the simplified strings. The list metric compares the the tokens.
	 * 
	 * @param metric
	 *            a list metric
	 * @param simplifier
	 *            a simplifier
	 * @param tokenizer
	 *            a tokenizer
	 * @return a new composite list metric
	 * 
	 * @throws NullPointerException
	 *             when either metric, simplifier or tokenizer are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric createForListMetric(Metric<List<String>> metric,
			Simplifier simplifier, Tokenizer tokenizer) {
		return new ForListWithSimplifier(metric, simplifier, tokenizer);
	}

	/**
	 * Creates a new composite string similarity metric. The tokenizer is used
	 * to tokenize the strings. The list metric compares the the tokens.
	 * 
	 * @param metric
	 *            a list metric
	 * @param tokenizer
	 *            a tokenizer
	 * @return a new composite similarity metric
	 * 
	 * @throws NullPointerException
	 *             when either metric or tokenizer are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric createForListMetric(Metric<List<String>> metric,
			Tokenizer tokenizer) {
		return new ForList(metric, tokenizer);
	}

	/**
	 * Creates a new composite string similarity metric.The tokenizer is used to
	 * tokenize the simplified strings. The set metric compares the the tokens.
	 * 
	 * @param metric
	 *            a list metric
	 * @param simplifier
	 *            a simplifier
	 * @param tokenizer
	 *            a tokenizer
	 * @return a new composite similarity metric
	 * 
	 * @throws NullPointerException
	 *             when either metric, simplifier or tokenizer are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric createForSetMetric(Metric<Set<String>> metric,
			Simplifier simplifier, Tokenizer tokenizer) {
		return new ForSetWithSimplifier(metric, simplifier, tokenizer);
	}

	/**
	 * Creates a new composite string similarity metric. The tokenizer is used
	 * to tokenize the strings. The set metric compares the the tokens.
	 * 
	 * @param metric
	 *            a set metric
	 * 
	 * @param tokenizer
	 *            a tokenizer
	 * @return a new composite similarity metric
	 * 
	 * @throws NullPointerException
	 *             when either metric or tokenizer are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric createForSetMetric(Metric<Set<String>> metric,
			Tokenizer tokenizer) {
		return new ForSet(metric, tokenizer);
	}

	/**
	 * Creates a new composite string similarity metric.The tokenizer is used to
	 * tokenize the simplified strings. The set metric compares the the tokens.
	 * 
	 * @param metric
	 *            a list metric
	 * @param simplifier
	 *            a simplifier
	 * @param tokenizer
	 *            a tokenizer
	 * @return a new composite similarity metric
	 * 
	 * @throws NullPointerException
	 *             when either metric, simplifier or tokenizer are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric createForMultisetMetric(
			Metric<Multiset<String>> metric, Simplifier simplifier,
			Tokenizer tokenizer) {
		return new ForMultisetWithSimplifier(metric, simplifier, tokenizer);
	}

	/**
	 * Creates a new composite string similarity metric. The tokenizer is used
	 * to tokenize the strings. The set metric compares the the tokens.
	 * 
	 * @param metric
	 *            a set metric
	 * 
	 * @param tokenizer
	 *            a tokenizer
	 * @return a new composite similarity metric
	 * 
	 * @throws NullPointerException
	 *             when either metric or tokenizer are null
	 * 
	 * @see StringMetricBuilder
	 * 
	 * @deprecated Use {@link StringMetricBuilder} in favor of directly
	 *             constructing a metric.
	 */
	@Deprecated
	public static StringMetric createForMultisetMetric(
			Metric<Multiset<String>> metric, Tokenizer tokenizer) {
		return new ForMultiset(metric, tokenizer);
	}

	static final class ForList implements StringMetric {
		private final Metric<List<String>> metric;
		private final Tokenizer tokenizer;

		ForList(Metric<List<String>> metric, Tokenizer tokenizer) {

			checkNotNull(metric);
			checkNotNull(tokenizer);

			this.metric = metric;
			this.tokenizer = tokenizer;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(tokenizer.tokenizeToList(a),
					tokenizer.tokenizeToList(b));
		}

		Metric<List<String>> getMetric() {
			return metric;
		}

		Tokenizer getTokenizer() {
			return tokenizer;
		}

		@Override
		public String toString() {
			return metric + " [" + tokenizer + "]";
		}
	}

	static final class ForListWithSimplifier implements StringMetric {
		private final Metric<List<String>> metric;
		private final Simplifier simplifier;
		private final Tokenizer tokenizer;

		ForListWithSimplifier(Metric<List<String>> metric,
				Simplifier simplifier, Tokenizer tokenizer) {

			checkNotNull(metric);
			checkNotNull(simplifier);
			checkNotNull(tokenizer);

			this.metric = metric;
			this.simplifier = simplifier;
			this.tokenizer = tokenizer;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(
					tokenizer.tokenizeToList(simplifier.simplify(a)),
					tokenizer.tokenizeToList(simplifier.simplify(b)));
		}

		Metric<List<String>> getMetric() {
			return metric;
		}

		Simplifier getSimplifier() {
			return simplifier;
		}

		Tokenizer getTokenizer() {
			return tokenizer;
		}

		@Override
		public String toString() {
			return metric + " [" + simplifier + " -> " + tokenizer + "]";
		}
	}

	static final class ForSet implements StringMetric {

		private final Metric<Set<String>> metric;
		private final Tokenizer tokenizer;

		ForSet(Metric<Set<String>> metric, Tokenizer tokenizer) {
			checkNotNull(metric);
			checkNotNull(tokenizer);

			this.metric = metric;
			this.tokenizer = tokenizer;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(tokenizer.tokenizeToSet(a),
					tokenizer.tokenizeToSet(b));
		}

		Metric<Set<String>> getMetric() {
			return metric;
		}

		Tokenizer getTokenizer() {
			return tokenizer;
		}

		@Override
		public String toString() {
			return metric + " [" + tokenizer + "]";
		}

	}

	static final class ForSetWithSimplifier implements StringMetric {

		private final Metric<Set<String>> metric;
		private final Simplifier simplifier;
		private final Tokenizer tokenizer;

		ForSetWithSimplifier(Metric<Set<String>> metric, Simplifier simplifier,
				Tokenizer tokenizer) {
			checkNotNull(metric);
			checkNotNull(simplifier);
			checkNotNull(tokenizer);

			this.metric = metric;
			this.simplifier = simplifier;
			this.tokenizer = tokenizer;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(
					tokenizer.tokenizeToSet(simplifier.simplify(a)),
					tokenizer.tokenizeToSet(simplifier.simplify(b)));
		}

		Metric<Set<String>> getMetric() {
			return metric;
		}

		Simplifier getSimplifier() {
			return simplifier;
		}

		Tokenizer getTokenizer() {
			return tokenizer;
		}

		@Override
		public String toString() {
			return metric + " [" + simplifier + " -> " + tokenizer + "]";
		}

	}

	static final class ForMultiset implements StringMetric {

		private final Metric<Multiset<String>> metric;
		private final Tokenizer tokenizer;

		ForMultiset(Metric<Multiset<String>> metric, Tokenizer tokenizer) {
			checkNotNull(metric);
			checkNotNull(tokenizer);

			this.metric = metric;
			this.tokenizer = tokenizer;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(tokenizer.tokenizeToMultiset(a),
					tokenizer.tokenizeToMultiset(b));
		}

		Metric<Multiset<String>> getMetric() {
			return metric;
		}

		Tokenizer getTokenizer() {
			return tokenizer;
		}

		@Override
		public String toString() {
			return metric + " [" + tokenizer + "]";
		}

	}

	static final class ForMultisetWithSimplifier implements StringMetric {

		private final Metric<Multiset<String>> metric;
		private final Simplifier simplifier;
		private final Tokenizer tokenizer;

		ForMultisetWithSimplifier(Metric<Multiset<String>> metric,
				Simplifier simplifier, Tokenizer tokenizer) {
			checkNotNull(metric);
			checkNotNull(simplifier);
			checkNotNull(tokenizer);

			this.metric = metric;
			this.simplifier = simplifier;
			this.tokenizer = tokenizer;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(
					tokenizer.tokenizeToMultiset(simplifier.simplify(a)),
					tokenizer.tokenizeToMultiset(simplifier.simplify(b)));
		}

		Metric<Multiset<String>> getMetric() {
			return metric;
		}

		Simplifier getSimplifier() {
			return simplifier;
		}

		Tokenizer getTokenizer() {
			return tokenizer;
		}

		@Override
		public String toString() {
			return metric + " [" + simplifier + " -> " + tokenizer + "]";
		}

	}

	static final class ForString implements StringMetric {
		private final Metric<String> metric;

		ForString(Metric<String> metric) {
			this.metric = metric;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(a, b);
		}

		@Override
		public String toString() {
			return metric.toString();
		}

		Metric<String> getMetric() {
			return metric;
		}

	}

	static final class ForStringWithSimplifier implements StringMetric {

		private final Metric<String> metric;

		private final Simplifier simplifier;

		ForStringWithSimplifier(Metric<String> metric, Simplifier simplifier) {
			checkNotNull(metric);
			checkNotNull(simplifier);

			this.metric = metric;
			this.simplifier = simplifier;
		}

		@Override
		public float compare(String a, String b) {
			return metric.compare(simplifier.simplify(a),
					simplifier.simplify(b));
		}

		Metric<String> getMetric() {
			return metric;
		}

		Simplifier getSimplifier() {
			return simplifier;
		}

		@Override
		public String toString() {
			return metric + " [" + simplifier + "]";
		}

	}

	private StringMetrics() {
		// Utility class.
	}

}


<|start_filename|>test/utility/TpcTest.java<|end_filename|>
package utility;

import org.junit.Test;

import java.util.ArrayList;
import java.util.List;


import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static utility.Tokenization.lowercaseSplit;

public class TpcTest {
	@Test
	public void similarity() {
		// We do not expect crisp 1 or 0 (because a continuous range 0..1 provide much more information).
		assertTrue(Tpc.similarity(lowercaseSplit("matches"), lowercaseSplit("tid"), lowercaseSplit("tid"), lowercaseSplit("teams")) == 1);  // Trivial
		assertTrue(Tpc.similarity(lowercaseSplit("TransactionHistoryArchive"), lowercaseSplit("ActualCost"), lowercaseSplit("UnitMeasureCode"), lowercaseSplit("UnitMeasure")) < 0.8);  // Different
		assertTrue(Tpc.similarity(lowercaseSplit("CountryLanguage"), lowercaseSplit("Language"), lowercaseSplit("Code"), lowercaseSplit("Country")) < 0.8);  // Different

		assertTrue(Tpc.similarity(lowercaseSplit("Stock"), lowercaseSplit("i_id"), lowercaseSplit("id"), lowercaseSplit("Item")) == 1);   // Abbreviations
		assertTrue(Tpc.similarity(lowercaseSplit("Stock"), lowercaseSplit("w_id"), lowercaseSplit("id"), lowercaseSplit("Item")) < 0.8);

		assertTrue(Tpc.similarity(lowercaseSplit("TRADE_REQUEST"), lowercaseSplit("CA_ID"), lowercaseSplit("ID"), lowercaseSplit("CUSTOMER_ACCOUNT")) == 1); // Two word abbreviations
		assertTrue(Tpc.similarity(lowercaseSplit("NEWS_XREF"), lowercaseSplit("NI_ID"), lowercaseSplit("ID"), lowercaseSplit("NEWS_ITEM")) == 1);
		assertTrue(Tpc.similarity(lowercaseSplit("NEWS_XREF"), lowercaseSplit("CO_ID"), lowercaseSplit("ID"), lowercaseSplit("NEWS_ITEM")) < 0.8);

		assertTrue(Tpc.similarity(lowercaseSplit("HOLDING_SUMMARY"), lowercaseSplit("AD_ID"), lowercaseSplit("ID"), lowercaseSplit("ADDRESS")) > 0.95);  // One word abbreviation to two letters
		assertTrue(Tpc.similarity(lowercaseSplit("INDUSTRY"), lowercaseSplit("SC_ID"), lowercaseSplit("ID"), lowercaseSplit("SECTOR")) > 0.95);

		assertTrue(Tpc.similarity(lowercaseSplit("connected"), lowercaseSplit("atom_id2"), lowercaseSplit("atom_id"), lowercaseSplit("atom")) > 0.95);      // Joker card
		assertTrue(Tpc.similarity(lowercaseSplit("connected"), lowercaseSplit("atom_id2_to"), lowercaseSplit("atom_id"), lowercaseSplit("atom")) < 1);
	}


	@Test(expected = IllegalArgumentException.class)
	public void emptyLists() throws Throwable {
		List<String> list = new ArrayList<>();

		Tpc.similarity(list, list, list, list);
	}

	@Test
	public void emptyStrings() throws Throwable {
		List<String> list = new ArrayList<>();
		list.add("");

		assertEquals(1.0, Tpc.similarity(list, list, list, list), 0.01);
	}

}


<|start_filename|>src/vendor/VendorFactory.java<|end_filename|>
package vendor;

public class VendorFactory {

	public static Vendor getVendor(String vendor) {
		switch (vendor) {
			case "MySQL":
				return new MySQL();
			case "PostgreSQL":
				return new PostgreSQL();
			case "Oracle":
				return new Oracle();
			case "Microsoft SQL Server":
				return new MSSQL();
			default:
				return new Default();
		}
	}
}


<|start_filename|>test/utility/HistogramTest.java<|end_filename|>
package utility;

import main.Column;
import main.Relationship;
import org.junit.Test;
import utility.Histogram;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static utility.Histogram.jaccard;

public class HistogramTest {

	@Test
	public void half() {
		double[] h1 = {0, 2};
		double[] h2 = {1, 3};
		assertEquals(0.3333, jaccard(h1, h2), 0.0001);
	}

	@Test
	public void duplicateMiddle() {
		double[] h1 = {0, 1, 2, 2, 3};
		double[] h2 = {1, 2};
		assertEquals(0.3333, jaccard(h1, h2), 0.0001);
	}

//		@Test
//	public void duplicateBeginning() {
//		double[] h1 = {1, 1, 2};
//		double[] h2 = {1, 3, 5};
//		assertEquals(0.6666, jaccard(h1, h2), 0.0001);
//	}

//	@Test
//	public void constant() {
//		double[] h1 = {1, 1, 1};
//		double[] h2 = {2, 2, 2};
//		System.out.println(jaccard(h1, h2));
//		assertEquals(0, jaccard(h1, h2), 0.0001);
//	}

	@Test
	public void random() {
		double[] h1 = {5, 8, 10, 11, 13, 14};
		double[] h2 = {1, 2, 3, 4, 5, 7};
		assertEquals(0.0714285714, jaccard(h1, h2), 0.0001);
	}




	@Test
	public void rel2abs_uniform() {
		double[] hist = {0.2, 0.2, 0.2, 0.2, 0.2};
		double[] expected = {0, 0.2, 0.4, 0.6, 0.8, 1};
		assertArrayEquals(expected, Histogram.rel2abs(hist, 0.0, 1.0), 0.0001);
	}

	@Test
	public void rel2abs_skewed() {
		double[] hist = {0.2, 0.8};
		double[] expected = {0, 0.2, 1};
		assertArrayEquals(expected, Histogram.rel2abs(hist, 0.0, 1.0), 0.0001);
	}

	@Test
	public void rel2abs_shifted() {
		double[] hist = {0.2, 0.8};
		double[] expected = {10, 10.2, 11};
		assertArrayEquals(expected, Histogram.rel2abs(hist, 10.0, 11.0), 0.0001);
	}

	@Test
	public void rel2abs_scaled() {
		double[] hist = {0.2, 0.8};
		double[] expected = {10, 10.4, 12};
		assertArrayEquals(expected, Histogram.rel2abs(hist, 10.0, 12.0), 0.0001);
	}

	@Test
	public void rel2abs_extremelySkewed() {
		double[] hist = {1, 0, 0};
		double[] expected = {10, 12, 12, 12};
		assertArrayEquals(expected, Histogram.rel2abs(hist, 10.0, 12.0), 0.1);
	}

//	@Test
//	public void rel2abs_constant() {
//		double[] hist = {1.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000,0.000};
//		System.out.println(Arrays.toString(Histogram.rel2abs(hist, 10.0, 12.0)));
//		double[] expected = {10, 12, 12, 12};
//		assertArrayEquals(expected, Histogram.rel2abs(hist, 10.0, 12.0), 0.1);
//	}

	@Test
	public void sandbox() {
		double[] hist = {1, 0, 0};
		double[] expected = {10, 12, 12, 12};
		String pkMin = "alzheimer";
		String pkMax = "zimbabwe";
		String fkMin = "dude";
		String fkMax = "mum";
//
		Relationship relationship = new Relationship();
		relationship.setFk(new Column("t1", "c1"));
		relationship.setPk(new Column("t1", "c1"));
		relationship.getPk().setTextMax("7.0");
		relationship.getPk().setTextMin("1.0");
		relationship.getFk().setTextMax("14224.0");
		relationship.getFk().setTextMin("6644.0");
		System.out.println(relationship.getRangeCoverage());

		assertArrayEquals(expected, Histogram.rel2abs(hist, 10.0, 12.0), 0.1);
	}



}


<|start_filename|>src/org/simmetrics/metrics/SmithWatermanGotoh.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static org.simmetrics.metrics.Math.max;

import org.simmetrics.StringMetric;
import org.simmetrics.metrics.functions.MatchMismatch;
import org.simmetrics.metrics.functions.Substitution;

/**
 * Applies the Smith-Waterman algorithm to calculate the similarity between two
 * strings. This implementation uses optimizations described in Osamu Gotoh
 * (1982). "An improved algorithm for matching biological sequences". Journal of
 * molecular biology 162: 705" and uses constant space and quadratic time.
 * 
 * <p>
 * This class is immutable and thread-safe if its substitution functions are.
 * 
 * @see NeedlemanWunch
 * @see SmithWaterman
 * @see <a
 *      href="https://en.wikipedia.org/wiki/Smith%E2%80%93Waterman_algorithm">Wikipedia
 *      - Smith-Waterman algorithm</a>
 *
 */
public final class SmithWatermanGotoh implements StringMetric {

	private static final Substitution MATCH_1_MISMATCH_MINUS_2 = new MatchMismatch(
			1.0f, -2.0f);

	private final float gapValue;

	private Substitution substitution;

	/**
	 * Constructs a new Smith Waterman metric. Gap penalty is -0.5, mismatch
	 * penalty -2.0 and a matching score 1.0.
	 * 
	 */
	public SmithWatermanGotoh() {
		this(-0.5f, MATCH_1_MISMATCH_MINUS_2);
	}

	/**
	 * Constructs a new Smith Waterman metric.
	 * 
	 * @param gapValue
	 *            a non-positive gap penalty
	 * @param substitution
	 *            a substitution function
	 */
	public SmithWatermanGotoh(float gapValue, Substitution substitution) {
		checkArgument(gapValue <= 0.0f);
		checkNotNull(substitution);
		this.gapValue = gapValue;
		this.substitution = substitution;
	}

	@Override
	public float compare(final String a, final String b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		float maxDistance = min(a.length(), b.length())
				* max(substitution.max(), gapValue);
		return smithWatermanGotoh(a, b) / maxDistance;
	}

	private float smithWatermanGotoh(final String s, final String t) {
		
		float[] v0 = new float[t.length()];
		float[] v1 = new float[t.length()];

		float max = v0[0] = max(0, gapValue, substitution.compare(s, 0, t, 0));

		for (int j = 1; j < v0.length; j++) {
			v0[j] = max(0, v0[j - 1] + gapValue,
					substitution.compare(s, 0, t, j));

			max = max(max, v0[j]);
		}

		// Find max
		for (int i = 1; i < s.length(); i++) {
			v1[0] = max(0, v0[0] + gapValue, substitution.compare(s, i, t, 0));

			max = max(max, v1[0]);

			for (int j = 1; j < v0.length; j++) {
				v1[j] = max(0, v0[j] + gapValue, v1[j - 1] + gapValue,
						v0[j - 1] + substitution.compare(s, i, t, j));

				max = max(max, v1[j]);
			}

			for (int j = 0; j < v0.length; j++) {
				v0[j] = v1[j];
			}
		}

		return max;
	}

	@Override
	public String toString() {
		return "SmithWatermanGotoh [substitution=" + substitution + ", gapValue="
				+ gapValue + "]";
	}
}


<|start_filename|>test/utility/TokenizationTest.java<|end_filename|>
package utility;

import org.junit.Test;

import java.util.Arrays;

import static main.Column.KEYWORDS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static utility.Tokenization.contains;
import static utility.Tokenization.lowercaseSplit;


public class TokenizationTest {

	@Test
	public void testCamelCased() {
		assertEquals(Arrays.asList("I", "Some", "Camel", "Cased", "String"), Tokenization.split("ISomeCamelCasedString"));
	}

	@Test
	public void testSnakeCased() {
		assertEquals(Arrays.asList("UNDERSCORED", "STRING"), Tokenization.split("UNDERSCORED_STRING"));
	}

	@Test
	public void testSnakeCasedSmall() {
		assertEquals(Arrays.asList("underscored", "string"), Tokenization.split("underscored_string"));
	}

	@Test
	public void testMixed() {
		assertEquals(Arrays.asList("camel", "Cased", "and", "UNDERSCORED"), Tokenization.split("camelCased_and_UNDERSCORED"));
	}

	@Test
	public void testSymbol() {
		assertEquals(Arrays.asList("_"), Tokenization.split("_"));
	}

	@Test
	public void testBattery() {
		assertEquals(Arrays.asList("lowercase"), Tokenization.split("lowercase"));
		assertEquals(Arrays.asList("Class"), Tokenization.split("Class"));
		assertEquals(Arrays.asList("My","Class"), Tokenization.split("MyClass"));
		assertEquals(Arrays.asList("HTML"), Tokenization.split("HTML"));
		assertEquals(Arrays.asList("PDF", "Loader"), Tokenization.split("PDFLoader"));
		assertEquals(Arrays.asList("A", "String"), Tokenization.split("AString"));
		assertEquals(Arrays.asList("Simple", "XML", "Parser"), Tokenization.split("SimpleXMLParser"));
		assertEquals(Arrays.asList("lower", "hyphen"), Tokenization.split("lower-hyphen"));
		assertEquals(Arrays.asList("99", "Bottles"), Tokenization.split("99Bottles"));
		assertEquals(Arrays.asList("GL", "11", "Version"), Tokenization.split("GL11Version"));
		assertEquals(Arrays.asList("May", "5"), Tokenization.split("May5"));
		assertEquals(Arrays.asList("BFG", "9000"), Tokenization.split("BFG9000"));
		assertEquals(Arrays.asList("Žluťoučký", "Kůň"), Tokenization.split("ŽluťoučkýKůň"));
		assertEquals(Arrays.asList("."), Tokenization.split("."));
		assertEquals(Arrays.asList(" "), Tokenization.split(" "));
		assertEquals(Arrays.asList("324", "A"), Tokenization.split(" 324A "));
	}

		@Test
	public void containsBatteryTest() {
		assertTrue(contains(lowercaseSplit("personId"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("person_id"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("PERSON_ID"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("id"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("ID"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("IdP"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("id_person"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("ID_PERSON"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("IDPerson"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("person_id2"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("personID2"), KEYWORDS));
		assertTrue(contains(lowercaseSplit("personId2"), KEYWORDS));

		assertFalse(contains(lowercaseSplit("personid"), KEYWORDS));
		assertFalse(contains(lowercaseSplit("PERSONID"), KEYWORDS));
		assertFalse(contains(lowercaseSplit("idperson"), KEYWORDS));
		assertFalse(contains(lowercaseSplit("IDPERSON"), KEYWORDS));
		assertFalse(contains(lowercaseSplit("x"), KEYWORDS));
		assertFalse(contains(lowercaseSplit("prefix_personid"), KEYWORDS));
		assertFalse(contains(lowercaseSplit("IDPERSON"), KEYWORDS));
	}
}


<|start_filename|>src/org/simmetrics/metrics/CosineSimilarity.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;
import static org.simmetrics.metrics.Math.union;
import static java.lang.Math.sqrt;

import org.simmetrics.MultisetDistance;
import org.simmetrics.MultisetMetric;

import com.google.common.collect.Multiset;

/**
 * Calculates the cosine similarity over two multisets. The similarity is
 * defined as the cosine of the angle between the multisets expressed as sparse
 * vectors.
 * <p>
 * <code>
 * similarity(a,b) = a·b / (||a|| * ||b||)
 * <br>
 * distance(a,b) = 1 - similarity(a,b)
 * </code>
 * 
 * <p>
 * The cosine similarity is identical to the Tanimoto coefficient, but unlike
 * Tanimoto the occurrence (cardinality) of an entry is taken into account. E.g.
 * {@code [hello, world]} and {@code [hello, world, hello, world]} would be
 * identical when compared with Tanimoto but are dissimilar when the cosine
 * similarity is used.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see TanimotoCoefficient
 * @see <a href="http://en.wikipedia.org/wiki/Cosine_similarity">Wikipedia
 *      Cosine similarity</a>
 * 
 * @param <T>
 *            type of the token
 */
public final class CosineSimilarity<T> implements MultisetMetric<T>, MultisetDistance<T> {

	@Override
	public float compare(Multiset<T> a, Multiset<T> b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		float dotProduct = 0;
		float magnitudeA = 0;
		float magnitudeB = 0;

		for (T entry : union(a, b).elementSet()) {
			float aCount = a.count(entry);
			float bCount = b.count(entry);

			dotProduct += aCount * bCount;
			magnitudeA += aCount * aCount;
			magnitudeB += bCount * bCount;
		}

		//  a·b / (||a|| * ||b||)
		return (float) (dotProduct / (sqrt(magnitudeA) * sqrt(magnitudeB)));
	}
	@Override
	public float distance(Multiset<T> a, Multiset<T> b) {
		return 1.0f - compare(a, b);
	}
	
	@Override
	public String toString() {
		return "CosineSimilarity";
	}



}


<|start_filename|>src/main/NullsExpected.java<|end_filename|>
package main;

import java.util.List;

public class NullsExpected {

    public static boolean containsKeyword(List<String> lowerCaseTokens) {
        for (String token : lowerCaseTokens) {
            // Makes a hierarchy
            if (token.contains("parent")) return true;
    
            // Is denormalized
            if (token.contains("3")) return true;
            if (token.contains("4")) return true;
            if (token.contains("5")) return true;
            if (token.contains("6")) return true;
            if (token.contains("7")) return true;
            if (token.contains("8")) return true;
            if (token.contains("9")) return true;
            if (token.contains("10")) return true;
            if (token.contains("11")) return true;
            if (token.contains("12")) return true;
    
            // Is a promise {close, cancel, return, refund, reason}
            if (token.contains("close")) return true;
            if (token.contains("cancel")) return true;
            if (token.contains("return")) return true;
            if (token.contains("refund")) return true;
            if (token.contains("reason")) return true;
        }

        return false;
    }
}


<|start_filename|>src/utility/MultiSet.java<|end_filename|>
package utility;

import java.util.HashMap;
import java.util.Map;

// A set, which keeps record of the value occurrence count.
public class MultiSet<K> extends HashMap<K, Integer>{
	private Map<K, Integer> objectCounts = new HashMap<>();
	private int totalCount = 0;     // Contains valid values only if items are added with the add() method (not with some of the methods of the HashMap like putIfAbsent...)

	// Constructor
	public MultiSet() {
		super();
	}

	public void add(K obj) {
		Integer count = objectCounts.get(obj);
		if (count == null) {
			objectCounts.put(obj, 1);
		} else {
			objectCounts.put(obj, ++count);
		}
		totalCount++;
	}

	public int count(K obj) {
		Integer count = objectCounts.get(obj);
		if (count == null) {
			return 0;
		} else {
			return count;
		}
	}

	public int totalCount() {
		return totalCount;
	}
}


<|start_filename|>src/org/simmetrics/SetMetric.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


package org.simmetrics;

import java.util.Collections;
import java.util.Set;

/**
 * Measures the similarity between two arbitrary sets containing elements of the
 * same type. The measurement results in a value between 0 and 1 inclusive. A
 * value of zero indicates that the sets are dissimilar, a value of 1 indicates
 * they are similar.
 * 
 * <p>
 * The elements in the set have to implement {@link Object#hashCode()} and
 * {@link Object#equals(Object)}.
 * <p>
 * The elements in the sets may not be null if metric does not support null
 * values.
 * <p>
 * The similarity measure should be consistent with equals such that
 * {@code a.equals(b) => compare(a,b) == 1.0}. 
 * <p>
 * The similarity measure should be reflexive such that
 * {@code compare(a,a) == 1.0}.
 * <p>
 * The similarity measure should be symmetric such that
 * {@code compare(a,b) == compare(b,a)}.
 * <p>
 * Implementations may not modify the contents of the set. Sets should be
 * treated as if wrapped by {@link Collections#unmodifiableSet(Set)}.
 * 
 * @param <T>
 *            the type of elements contained in the sets
 *
 */
public interface SetMetric<T> extends Metric<Set<T>> {
	/**
	 * Measures the similarity between sets a and b. The measurement results in
	 * a value between 0 and 1 inclusive. A value of zero indicates that the
	 * sets are dissimilar, a value of 1 indicates they are similar.
	 * <p>
	 * Results are undefined if {@code set1} and {@code set2} are sets based on
	 * different equivalence relations (as {@code HashSet}, {@code TreeSet}, and
	 * the keySet of an {@code IdentityHashMap} all are).
	 * 
	 * @param a
	 *            set a to compare
	 * @param b
	 *            set b to compare
	 * @return a value between 0 and 1 inclusive indicating similarity
	 * @throws NullPointerException
	 *             when either a or b is null
	 */
	@Override
	public float compare(Set<T> a, Set<T> b);

}


<|start_filename|>src/org/simmetrics/simplifiers/Caverphone1.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */
package org.simmetrics.simplifiers;

import static com.google.common.base.Preconditions.checkNotNull;

/**
 * Encodes a string into a Caverphone 1.0 value.
 * <p>
 * This is an algorithm created by the Caversham Project at the University of
 * Otago. It implements the Caverphone 1.0 algorithm.
 * <p>
 * This class is immutable and thread-safe.
 * </p>
 *
 * @see <a href="http://en.wikipedia.org/wiki/Caverphone">Wikipedia -
 *      Caverphone</a>
 * @see <a
 *      href="http://caversham.otago.ac.nz/files/working/ctp150804.pdf">Caverphone
 *      2.0 specification</a>
 * @see org.apache.commons.codec.language.Caverphone1
 * 
 * @deprecated will be removed due to a lack of a good use case
 *
 */
@Deprecated
public final class Caverphone1 implements Simplifier {

	private final org.apache.commons.codec.language.Caverphone1 simplifier = new org.apache.commons.codec.language.Caverphone1();

	@Override
	public String simplify(String input) {
		checkNotNull(input);
		return simplifier.encode(input);
	}
	
	@Override
	public String toString() {
		return "Caverphone1";
	}

}


<|start_filename|>src/org/simmetrics/metrics/Jaro.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

package org.simmetrics.metrics;

import static java.lang.Math.max;
import static java.lang.Math.min;

import org.simmetrics.StringDistance;
import org.simmetrics.StringMetric;

/**
 * Calculates the Jaro distance (similarity) over two strings.
 * <p>
 * <code>
 * similarity(a,b) = jaro(a,b)
 * <br>
 * distance(a,b) = 1 - similarity(a,b)
 * </code>
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see <a
 *      href="http://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance">Wikipedia
 *      - Jaro-Winkler distance</a>
 * @see JaroWinkler
 *
 *
 *
 */
public final class Jaro implements StringMetric, StringDistance {
	
	@Override
	public float distance(String a, String b) {
		return 1.0f - compare(a, b);
	}
	
	@Override
	public float compare(final String a, final String b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		// Intentional integer division to round down.
		final int halfLength = max(0, max(a.length(), b.length()) / 2 - 1);

		final char[] charsA = a.toCharArray();
		final char[] charsB = b.toCharArray();
		final int[] commonA = getCommonCharacters(charsA, charsB, halfLength);
		final int[] commonB = getCommonCharacters(charsB, charsA, halfLength);

		// commonA and commonB will always contain the same multi-set of
		// characters. Because getCommonCharacters has been optimized, commonA
		// and commonB are -1-padded. So in this loop we count transposition
		// and use commonCharacters to determine the length of the multi-set.
		float transpositions = 0;
		int commonCharacters = 0;
		for (int length = commonA.length; commonCharacters < length
				&& commonA[commonCharacters] > -1; commonCharacters++) {
			if (commonA[commonCharacters] != commonB[commonCharacters]) {
				transpositions++;
			}
		}

		if (commonCharacters == 0) {
			return 0.0f;
		}

		float aCommonRatio = commonCharacters / (float) a.length();
		float bCommonRatio = commonCharacters / (float) b.length();
		float transpositionRatio = (commonCharacters - transpositions / 2.0f)
				/ commonCharacters;

		return (aCommonRatio + bCommonRatio + transpositionRatio) / 3.0f;
	}

	/*
	 * Returns an array of characters from a within b. A character in b is
	 * counted as common when it is within separation distance from the position
	 * in a.
	 */
	private static int[] getCommonCharacters(final char[] charsA,
			final char[] charsB, final int separation) {
		final int[] common = new int[min(charsA.length, charsB.length)];
		final boolean[] matched = new boolean[charsB.length];

		// Iterate of string a and find all characters that occur in b within
		// the separation distance. Mark any matches found to avoid
		// duplicate matchings.
		int commonIndex = 0;
		for (int i = 0, length = charsA.length; i < length; i++) {
			final char character = charsA[i];
			final int index = indexOf(character, charsB, i - separation, i
					+ separation + 1, matched);
			if (index > -1) {
				common[commonIndex++] = character;
				matched[index] = true;
			}
		}

		if (commonIndex < common.length) {
			common[commonIndex] = -1;
		}

		// Both invocations will yield the same multi-set terminated by -1, so
		// they can be compared for transposition without making a copy.
		return common;
	}

	/*
	 * Search for character in buffer starting at fromIndex to toIndex - 1.
	 * 
	 * Returns -1 when not found.
	 */
	private static int indexOf(char character, char[] buffer, int fromIndex,
			int toIndex, boolean[] matched) {

		// compare char with range of characters to either side
		for (int j = max(0, fromIndex), length = min(toIndex, buffer.length); j < length; j++) {
			// check if found
			if (buffer[j] == character && !matched[j]) {
				return j;
			}
		}

		return -1;
	}

	@Override
	public String toString() {
		return "Jaro";
	}
}


<|start_filename|>src/org/simmetrics/metrics/NeedlemanWunch.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static org.simmetrics.metrics.Math.min;

import java.util.Objects;

import org.simmetrics.StringMetric;
import org.simmetrics.metrics.functions.MatchMismatch;
import org.simmetrics.metrics.functions.Substitution;

/**
 * Applies the Needleman-Wunsch algorithm to calculate the similarity
 * between two strings. This implementation uses linear space.
 * <p>
 * This class is immutable and thread-safe if its substitution function is.
 * 
 * @see SmithWatermanGotoh
 * @see SmithWaterman
 * @see <a
 *      href="https://en.wikipedia.org/wiki/Needleman%E2%80%93Wunsch_algorithm">Wikipedia
 *      - Needleman-Wunsch algorithm</a>
 */
public final class NeedlemanWunch implements StringMetric {

	private static final Substitution MATCH_0_MISMATCH_1 = new MatchMismatch(
			0.0f, -1.0f);

	private final Substitution substitution;

	private final float gapValue;

	/**
	 * Constructs a new Needleman-Wunch metric. Uses an gap of <code>-2.0</code>
	 * a <code>-1.0</code> substitution penalty for mismatches, <code>0</code>
	 * for matches.
	 * 
	 */
	public NeedlemanWunch() {
		this(-2.0f, MATCH_0_MISMATCH_1);
	}

	/**
	 * Constructs a new Needleman-Wunch metric.
	 * 
	 * @param gapValue
	 *            a non-positive penalty for gaps
	 * @param substitution
	 *            a substitution function for mismatched characters
	 */
	public NeedlemanWunch(float gapValue, Substitution substitution) {
		checkArgument(gapValue <= 0.0f);
		checkNotNull(substitution);
		this.gapValue = gapValue;
		this.substitution = substitution;
	}

	@Override
	public float compare(String a, String b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		float maxDistance = max(a.length(), b.length())
				* max(substitution.max(), gapValue);
		float minDistance = max(a.length(), b.length())
				* min(substitution.min(), gapValue);

		return (-needlemanWunch(a, b) - minDistance)
				/ (maxDistance - minDistance);

	}

	private float needlemanWunch(final String s, final String t) {

		if (Objects.equals(s, t)) {
			return 0;
		}

		if (s.isEmpty()) {
			return -gapValue * t.length();
		}
		if (t.isEmpty()) {
			return -gapValue * s.length();
		}
		
		final int n = s.length();
		final int m = t.length();

		// We're only interested in the alignment penalty between s and t
		// and not their actual alignment. This means we don't have to backtrack
		// through the n-by-m matrix and can safe some space by reusing v0 for
		// row i-1.
		float[] v0 = new float[m + 1];
		float[] v1 = new float[m + 1];

		for (int j = 0; j <= m; j++) {
			v0[j] = j;
		}

		for (int i = 1; i <= n; i++) {
			v1[0] = i;

			for (int j = 1; j <= m; j++) {
				v1[j] = min(
						v0[j]     - gapValue, 
						v1[j - 1] - gapValue, 
						v0[j - 1] - substitution.compare(s, i - 1, t, j - 1));
			}
			
			final float[] swap = v0; v0 = v1; v1 = swap;

		}
		
		// Because we swapped the results are in v0.
		return v0[m];
	}

	@Override
	public String toString() {
		return "NeedlemanWunch [costFunction=" + substitution + ", gapCost="
				+ gapValue + "]";
	}

}


<|start_filename|>src/utility/Tpc.java<|end_filename|>
package utility;

import main.Setting;
import org.simmetrics.metrics.StringMetrics;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class Tpc {

	private static final org.simmetrics.StringMetric metric = StringMetrics.jaroWinkler(); // In this case, jaroWinkler works better than levenshtein, lcs (by a tiny part), jaro, smithWaterman, overlapCoefficient or qGramsDistance (not so bad)

	// FK name generally composes of following token types:
	//  1) PK table name
	//  2) PK column name
	//  3) FK table name
	//  4) Relationship name (e.g. "working", "studying", "to", "from",...)
	//  5) Keyword saying that it is a key (e.g. "id", "fk",...)
	// We just attempt to match the tokens in the FK column name to tokens enlisted above.
	// Note however, that each single token can be used maximally once -> use Hungarian algorithm.
	public static Double similarity(List<String> fkTable, List<String> fk, List<String> pk, List<String> pkTable) {
		// Input parameter validation
		if (fkTable.isEmpty() || fk.isEmpty() || pk.isEmpty() || pkTable.isEmpty()) {
			throw new IllegalArgumentException("The input lists should never be empty");
		}

		// Initialize two lists of tokens
		List<String> fkTokens = fk;
		List<String> otherTokens = new ArrayList<>();
		//      otherTokens.add(getAbbreviation(fkTable));  // Only in the abbreviated form for fkTable!
		otherTokens.addAll(pk);
		otherTokens.addAll(pkTable);
		otherTokens.add(getAbbreviation(pkTable));

		// Add KEYWORD - a single joker card, which can represent any single token from KEYWORDS list
		otherTokens.add("KEYWORD");

		// Get cost matrix between each token in fkTokens and each token in otherTokens
		double[][] cost = new double[fkTokens.size()][otherTokens.size()];
		for (int i = 0; i < fkTokens.size(); i++) {
			for (int j = 0; j < otherTokens.size(); j++) {
				cost[i][j] = 1 - similarity(fkTokens.get(i), otherTokens.get(j), otherTokens);
			}
		}

		// Get optimal matching
		Hungarian hungarian = new Hungarian(cost);
		int[] result = hungarian.execute();

		// Calculate similarity
		double similarity = solutionSimilarity(fkTokens, otherTokens, result);

		// Print solution
//      System.out.print("Other tokens: ");
//      for (String token : otherTokens) {
//          System.out.print(token + " ");
//      }
//      System.out.print("\nMatch cost: ");
//      printArray(cost);
//      System.out.println("Foreign column tokens: " + fk);
//      System.out.print("Matched tokens: ");
//      for (int i : result) {
//          System.out.print(otherTokens.get(i) + " ");
//      }
//      System.out.println();
//      System.out.println("Similarity is: " + similarity);
//      System.out.println();

		return similarity;
	}

	private static double similarity(String a, String b, List<String> otherTokens) {
		double sim = metric.compare(a, b);
		double sim2 = asymmetricAbbreviationSimilarity(a, b, otherTokens);
		double result = Math.max(sim, sim2);
		return result;
	}

	// Returns concatenated initial letters from tokens.
	// This is used frequently in tpc schemas.
	private static String getAbbreviation(List<String> tokens) {
		String result = "";
		for (String token : tokens) {
			if (token.length()>0) {
				result += token.substring(0, 1);
			}
		}

		return result;
	}

	// We are interested only in the abbreviation/joker card in the FK
	private static double asymmetricAbbreviationSimilarity(String a, String b, List<String> otherTokens) {
		// Joker card
		if ("KEYWORD".equals(b) && Setting.KEYWORD_RELATIONSHIP.contains(a) && !otherTokens.contains(a)) return 0.95;

		// Abbreviation to two letters (e.g. address -> ad)
		if (a.length() == 2 && b.length() >= 2 && a.charAt(0) == b.charAt(0) && a.charAt(1) == b.charAt(1)) {
			return 0.95;    // We penalize that a bit
		}

		// Another abbreviation to two letters (e.g. sector -> sc)
		if (a.length() == 2 && b.length() >= 3 && a.charAt(0) == b.charAt(0) && a.charAt(1) == b.charAt(2)) {
			return 0.95;
		}

		// Employ related terms (synonyms, hypernyms and hyponyms). Unfortunately, it is a hard match/mismatch
		if (Setting.SYNONYM_PERSON.contains(a) && Setting.SYNONYM_PERSON.contains(b)) {
			return 0.95;
		}

		if (Setting.SYNONYM_COUNTRY.contains(a) && Setting.SYNONYM_COUNTRY.contains(b)) {
			return 0.95;
		}

		if (Setting.SYNONYM_PRODUCT.contains(a) && Setting.SYNONYM_PRODUCT.contains(b)) {
			return 0.95;
		}

		if (Setting.SYNONYM_CITY.contains(a) && Setting.SYNONYM_CITY.contains(b)) {
			return 0.95;
		}

		return 0.0;
	}

	private static void printArray(double matrix[][]) {
		for (double[] row : matrix)
			System.out.println(Arrays.toString(row));
	}

	private static double solutionSimilarity(List<String> fkTokens, List<String> otherTokens, int[] matching) {
		double similarity = 0;
		double tokenCount = fkTokens.size();

		for (int i = 0; i < matching.length; i++) {
			if (matching[i] >= 0) { // If unmatched, we "increment" similarity by 0
				similarity += similarity(fkTokens.get(i), otherTokens.get(matching[i]), otherTokens);
			}
		}

		return similarity / tokenCount;
	}
}


<|start_filename|>src/main/CompoundRelationship.java<|end_filename|>
package main;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;

// Converts a set of simple foreign key constraints to compound foreign key constraints.
public class CompoundRelationship {

	private final List<Relationship> relationships;
	private final List<String> fkColumns = new ArrayList<>();
	private final List<String> pkColumns = new ArrayList<>();
	private final String fkTable;
	private final String pkTable;

	private CompoundRelationship(List<Relationship> relationships) {
		this.relationships = relationships;
		fkTable = relationships.get(0).getFkTableName();
		pkTable = relationships.get(0).getPkTableName();
		relationships.forEach(w -> fkColumns.add(w.getFk().getName()));
		relationships.forEach(w -> pkColumns.add(w.getPk().getName()));
	}

	private CompoundRelationship(Relationship relationship) {
		relationships = new ArrayList<>();
		relationships.add(relationship);
		fkTable = relationship.getFkTableName();
		pkTable = relationship.getPkTableName();
		fkColumns.add(relationship.getFk().getName());
		pkColumns.add(relationship.getPk().getName());
	}

	public static List<CompoundRelationship> buildFrom(List<Relationship> relationships) {
		List<Relationship> compound = new ArrayList<>();
		List<CompoundRelationship> result = new ArrayList<>();

		// If the PK is compound, build a compound FK constraint. Otherwise build a simple FK constraint.
		for (Relationship relationship : relationships) {
			if (relationship.isEstimatedFk()) {
				if (relationship.getPk().getTable().getBestAttemptPk().size() > 1) {
					compound.add(relationship);
				} else {
					result.add(new CompoundRelationship(relationship));
				}
			}
		}
		result.addAll(getCompound(compound));

		return result;
	}


	private static List<CompoundRelationship> getCompound(List<Relationship> relationships) {
		List<CompoundRelationship> result = new ArrayList<>();

		// Sort by FK and PK table names
		relationships.sort(Comparator.comparing(Relationship::getPkTableName).thenComparing(Relationship::getFkTableName));

		// Loop
		String previousPkTable = "";
		String previousFkTable = "";
		List<Relationship> relationshipList = new ArrayList<>();
		for (Relationship relationship : relationships) {
			if (!previousFkTable.isEmpty() && !(relationship.getFkTableName().equals(previousFkTable) && relationship.getPkTableName().equals(previousPkTable))) {
				result.add(new CompoundRelationship(relationshipList));
				relationshipList.clear();
			}

			// Update
			relationshipList.add(relationship);
			previousPkTable = relationship.getPkTableName();
			previousFkTable = relationship.getFkTableName();
		}

		// Add the last item
		if (!previousFkTable.isEmpty()) {
			result.add(new CompoundRelationship(relationshipList));
		}

		return result;
	}

	public boolean isEstimatedFk() {
		return relationships.stream().allMatch(w -> w.isEstimatedFk());
	}

	public boolean isForeignKey() {
		return relationships.stream().allMatch(w -> w.isForeignKey());
	}


	////// Boring getters
	public String getFkTable() {
		return fkTable;
	}

	public String getPkTable() {
		return pkTable;
	}

	public List<String> getFkColumns() {
		return fkColumns;
	}

	public List<String> getPkColumns() {
		return pkColumns;
	}
}


<|start_filename|>gui/utility/PrefixSelectionCustomizer.java<|end_filename|>
package utility;

/**
 * Copyright (c) 2015, ControlsFX
 * All rights reserved.
 * <p>
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * * Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 * * Neither the name of ControlsFX, any associated website, nor the
 * names of its contributors may be used to endorse or promote products
 * derived from this software without specific prior written permission.
 * <p>
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL CONTROLSFX BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */


import javafx.collections.ObservableList;
import javafx.event.EventHandler;
import javafx.scene.control.ChoiceBox;
import javafx.scene.control.ComboBox;
import javafx.scene.control.Control;
import javafx.scene.input.KeyCode;
import javafx.scene.input.KeyEvent;
import javafx.util.StringConverter;

import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;

/**
 * <p>This utility class can be used to customize a {@link ChoiceBox} or
 * {@link ComboBox} and enable the "prefix selection" feature. This will enable
 * the user to type letters or digits on the keyboard while the {@link ChoiceBox}
 * or {@link ComboBox} has the focus. The {@link ChoiceBox} or {@link ComboBox}
 * will attempt to select the first item it can find with a matching prefix ignoring
 * case.
 *
 * <p>This feature is available natively on the Windows combo box control, so many
 * users have asked for it. There is a feature request to include this feature
 * into JavaFX (<a href="https://javafx-jira.kenai.com/browse/RT-18064">Issue RT-18064</a>).
 * The class is published as part of ContorlsFX to allow testing and feedback.
 *
 * <h3>Example</h3>
 *
 * <p>Let's look at an example to clarify this. The combo box offers the items
 * ["Aaaaa", "Abbbb", "Abccc", "Abcdd", "Abcde"]. The user now types "abc" in
 * quick succession (and then stops typing). The combo box will select a new entry
 * on every key pressed. The first entry it will select is "Aaaaa" since it is the
 * first entry that starts with an "a" (case ignored). It will then select "Abbbb",
 * since this is the first entry that started with "ab" and will finally settle for
 * "Abccc".
 *
 * <ul><table>
 *   <tr><th>Keys typed</th><th>Element selected</th></tr>
 *   <tr><td>a</td><td>Aaaaa<td></tr>
 *   <tr><td>aaa</td><td>Aaaaa<td></tr>
 *   <tr><td>ab</td><td>Abbbb<td></tr>
 *   <tr><td>abc</td><td>Abccc<td></tr>
 *   <tr><td>xyz</td><td>-<td></tr>
 * </table></ul>
 *
 * <h3>Usage</h3>
 *
 * <p>A common use case is to customize a {@link ChoiceBox} or {@link ComboBox}
 * that has been loaded as part of an FXML. In this case you can use the utility
 * methods {@link #customize(ChoiceBox)} or {@link #customize(ComboBox)}. This
 * will install a {@link EventHandler} that monitors the {@link KeyEvent}
 * events.
 *
 * It is not perfect - for example, it does not automatically scroll to the item, but it is better than nothing.
 * For alternatives, see:
 *  http://stackoverflow.com/questions/13362607/combobox-jump-to-typed-char
 */
public class PrefixSelectionCustomizer {
	private static final String SELECTION_PREFIX_STRING = "selectionPrefixString";
	private static final Object SELECTION_PREFIX_TASK = "selectionPrefixTask";

	private static EventHandler<KeyEvent> handler = new EventHandler<KeyEvent>() {
		private ScheduledExecutorService executorService;

		@Override
		public void handle(KeyEvent event) {
			keyPressed(event);
		}

		private <T> void keyPressed(KeyEvent event) {
			KeyCode code = event.getCode();
			if (code.isLetterKey() || code.isDigitKey() || code == KeyCode.SPACE) {
				String letter = code.impl_getChar();
				if (event.getSource() instanceof ComboBox) {
					ComboBox<T> comboBox = (ComboBox<T>) event.getSource();
					T item = getEntryWithKey(letter, comboBox.getConverter(), comboBox.getItems(), comboBox);
					if (item != null) {
						comboBox.setValue(item);
					}
				} else if (event.getSource() instanceof ChoiceBox) {
					ChoiceBox<T> choiceBox = (ChoiceBox<T>) event.getSource();
					T item = getEntryWithKey(letter, choiceBox.getConverter(), choiceBox.getItems(), choiceBox);
					if (item != null) {
						choiceBox.setValue(item);
					}
				}
			}
		}

		private <T> T getEntryWithKey(String letter, StringConverter<T> converter, ObservableList<T> items, Control control) {
			T result = null;

			// The converter is null by default for the ChoiceBox. The ComboBox has a default converter
			if (converter == null) {
				converter = new StringConverter<T>() {
					@Override
					public String toString(T t) {
						return t == null ? null : t.toString();
					}

					@Override
					public T fromString(String string) {
						return null;
					}
				};
			}

			String selectionPrefixString = (String) control.getProperties().get(SELECTION_PREFIX_STRING);
			if (selectionPrefixString == null) {
				selectionPrefixString = letter.toUpperCase();
			} else {
				selectionPrefixString += letter.toUpperCase();
			}
			control.getProperties().put(SELECTION_PREFIX_STRING, selectionPrefixString);

			for (T item : items) {
				String string = converter.toString(item);
				if (string != null && string.toUpperCase().startsWith(selectionPrefixString)) {
					result = item;
					break;
				}
			}

			ScheduledFuture<?> task = (ScheduledFuture<?>) control.getProperties().get(SELECTION_PREFIX_TASK);
			if (task != null) {
				task.cancel(false);
			}
			task = getExecutorService().schedule(
					() -> control.getProperties().put(SELECTION_PREFIX_STRING, ""), 500, TimeUnit.MILLISECONDS);
			control.getProperties().put(SELECTION_PREFIX_TASK, task);

			return result;
		}

		private ScheduledExecutorService getExecutorService() {
			if (executorService == null) {
				executorService = Executors.newScheduledThreadPool(1,
						runnabble -> {
							Thread result = new Thread(runnabble);
							result.setDaemon(true);
							return result;
						});
			}
			return executorService;
		}

	};

	/**
	 * This will install an {@link EventHandler} that monitors the
	 * {@link KeyEvent} events to enable the "prefix selection" feature.
	 * The {@link EventHandler} will only be installed if the {@link ComboBox}
	 * is <b>not</b> editable.
	 *
	 * @param comboBox
	 *            The {@link ComboBox} that should be customized
	 *
	 * @see PrefixSelectionCustomizer
	 */
	public static void customize(ComboBox<?> comboBox) {
		if (!comboBox.isEditable()) {
			comboBox.addEventHandler(KeyEvent.KEY_PRESSED, handler);
		}
		comboBox.editableProperty().addListener((o, oV, nV) -> {
			if (!nV) {
				comboBox.addEventHandler(KeyEvent.KEY_PRESSED, handler);
			} else {
				comboBox.removeEventHandler(KeyEvent.KEY_PRESSED, handler);
			}
		});
	}

	/**
	 * This will install an {@link EventHandler} that monitors the
	 * {@link KeyEvent} events to enable the "prefix selection" feature.
	 *
	 * @param choiceBox
	 *            The {@link ChoiceBox} that should be customized
	 *
	 * @see PrefixSelectionCustomizer
	 */
	public static void customize(ChoiceBox<?> choiceBox) {
		choiceBox.addEventHandler(KeyEvent.KEY_PRESSED, handler);
	}

}

<|start_filename|>src/org/simmetrics/metrics/OverlapCoefficient.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;

import static org.simmetrics.metrics.Math.intersection;
import static java.lang.Math.min;

import java.util.Set;

import org.simmetrics.SetDistance;
import org.simmetrics.SetMetric;

/**
 * The overlap coefficient measures the overlap between two sets. The similarity
 * is defined as the size of the intersection divided by the smaller of the size
 * of the two sets.
 * <p>
 * <code>
 * similarity(q,r) = ∣q ∩ r∣ / min{∣q∣, ∣r∣} <br>
 * distance(q,r) = 1 - similarity(q,r)
 * </code>
 * <p>
 * Unlike the generalized overlap coefficient the occurrence (cardinality) of an
 * entry is not taken into account. E.g. {@code [hello, world]} and
 * {@code [hello, world, hello, world]} would be identical when compared with
 * the overlap coefficient but are dissimilar when the generalized version is
 * used.
 * <p>
 * Similar to the generalized Jaccard similarity which divides the intersection
 * by the union of two multisets.
 * <p>
 * Similar to the dice coefficient which divides the shared information
 * (intersection) by sum of cardinalities.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @param <T>
 *            type of the token
 * 
 * @see GeneralizedOverlapCoefficient
 * @see Jaccard
 * @see Dice
 * @see <a href="http://en.wikipedia.org/wiki/Overlap_coefficient">Wikipedia -
 *      Overlap Coefficient</a>
 */
public final class OverlapCoefficient<T> implements SetMetric<T>, SetDistance<T> {

	@Override
	public float distance(Set<T> a, Set<T> b) {
		return 1 - compare(a, b);
	}
	
	@Override
	public float compare(Set<T> a, Set<T> b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		// ∣q ∩ r∣ / min{∣q∣, ∣r∣}
		return intersection(a, b).size() / (float) min(a.size(), b.size());
	}

	@Override
	public String toString() {
		return "OverlapCoefficient";
	}

}


<|start_filename|>src/org/simmetrics/metrics/functions/Substitution.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


package org.simmetrics.metrics.functions;

/**
 * A substitution function assigns value to the substitution of one character
 * for another matching against another string.
 * <p>
 * A positive value indicates the characters are similar, a negative value
 * indicates they are dissimilar.
 * 
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see Gap
 * @see <a
 *      href="https://en.wikipedia.org/wiki/Smith%E2%80%93Waterman_algorithm">Wikipedia
 *      - Smith–Waterman algorithm</a>
 *
 */
public interface Substitution {
	/**
	 * Rates the substitution of character <code>a.getChar(aIndex)</code>
	 * against <code>b.getChar(bIndex)</code>.
	 * 
	 * @param a
	 *            a string to check in
	 * @param aIndex
	 *            index of character at string <code>a</code> to compare
	 * @param b
	 *            another string to check in
	 * @param bIndex
	 *            index of character at string <code>b</code> to compare
	 * @return a score indicating the characters (dis) similarity
	 */
	public float compare(String a, int aIndex, String b, int bIndex);

	/**
	 * Returns the maximum value a gap can have
	 * 
	 * @return the maximum value a gap can have
	 */
	public float max();

	/**
	 * Returns the minimum value a gap can have
	 * 
	 * @return the minimum value a gap can have
	 */
	public float min();
}


<|start_filename|>src/utility/Multiplication.java<|end_filename|>
package utility;

public class Multiplication {

  public static double[] elementWise(double[] vec1, double[] vec2) {
	  // Input parameter control
	  if (vec1.length != vec2.length) throw new RuntimeException("Vectors have different lengths: " + vec1.length + " and " + vec2.length);

	  double[] result = new double[vec1.length];
	  for (int i = 0; i < vec1.length; i++) {
		  result[i] = vec1[i] * vec2[i];
	  }

	  return result;
  }
}


<|start_filename|>test/example/DemoTest.java<|end_filename|>
package example;

import org.junit.Test;


public class DemoTest {

	@Test
	public void shouldNotThrowAnyException() {
		Demo.main(null);
	}

}


<|start_filename|>src/org/simmetrics/metrics/GeneralizedJaccard.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;

import static org.simmetrics.metrics.Math.intersection;

import org.simmetrics.MultisetDistance;
import org.simmetrics.MultisetMetric;

import com.google.common.collect.Multiset;

/**
 * Calculates the generalized Jaccard distance and similarity coefficient over
 * two multisets. The similarity is defined as the size of the intersection
 * divided by the size of the union of the sample sets. The distance is obtained
 * by subtracting the Jaccard coefficient from 1.
 * <p>
 * <code>
 * similarity(a,b) = ∣a ∩ b∣ / ∣a ∪ b∣
 * <br>
 * distance(a,b) = 1 - similarity(a,b)
 * </code>
 * <p>
 * When <code>∣a ∪ b∣</code> is empty the multisets have no elements in common.
 * In this case the similarity is 0 by definition.
 * <p>
 * Unlike the Jaccard index the occurrence (cardinality) of an entry is taken
 * into account. E.g. {@code [hello, world]} and
 * {@code [hello, world, hello, world]} would be identical when compared with
 * the Jaccard index but are dissimilar when the generalized version is used.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see Jaccard
 * @see <a href="http://en.wikipedia.org/wiki/Jaccard_index">Wikipedia - Jaccard
 *      index</a>
 * 
 * @param <T>
 *            type of the token
 * 
 */
public final class GeneralizedJaccard<T> implements MultisetMetric<T>,
		MultisetDistance<T> {
	@Override
	public float compare(Multiset<T> a, Multiset<T> b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		final int intersection = intersection(a, b).size();

		// ∣a ∩ b∣ / ∣a ∪ b∣
		// Implementation note: The size of the union of two sets is equal to
		// the size of both sets minus the duplicate elements.
		return intersection / (float) (a.size() + b.size() - intersection);
	}

	@Override
	public float distance(Multiset<T> a, Multiset<T> b) {
		return 1.0f - compare(a, b);
	}

	@Override
	public String toString() {
		return "GeneralizedJaccard";
	}

}


<|start_filename|>test/main/DoppelgangerTest.java<|end_filename|>
package main;

import main.Column;
import main.Table;
import org.junit.Before;
import org.junit.Test;

import static main.Doppelganger.isDoppelganger;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

public class DoppelgangerTest {

	private Table table;

	@Before
	public void init() {
		table = new Table("schema", "table");
		table.getColumnList().add(new Column("table", "atom_id1"));
		table.getColumnList().add(new Column("table", "atom_id2"));
		table.getColumnList().add(new Column("table", "chair_id"));
		table.getColumnList().add(new Column("table", "chair_id2"));
		table.getColumnList().add(new Column("table", "busIdA"));
		table.getColumnList().add(new Column("table", "busIdB"));
		table.getColumnList().add(new Column("table", "cop1_id"));
		table.getColumnList().add(new Column("table", "cop2_id"));
		table.getColumnList().add(new Column("table", "AddressLine1"));
		table.getColumnList().add(new Column("table", "AddressLine2"));
		table.getColumnList().add(new Column("table", "fish"));
		table.getColumnList().add(new Column("table", "fish_2"));
		table.getColumnList().add(new Column("table", "unique"));
		table.getColumnList().add(new Column("table", "kid"));
		table.getColumnList().add(new Column("table", "kids"));
		table.getColumnList().add(new Column("table", "molecule_1"));
		table.getColumnList().add(new Column("table", "molecule_2"));
		table.getColumnList().add(new Column("table", "molecule_3"));
		table.getColumnList().add(new Column("table", "molecule_4"));
		table.getColumnList().add(new Column("table", "molecule_5"));
		table.getColumnList().add(new Column("table", "molecule_6"));
		table.getColumnList().add(new Column("table", "molecule_7"));
		table.getColumnList().add(new Column("table", "molecule_8"));
		table.getColumnList().add(new Column("table", "molecule_9"));
		table.getColumnList().add(new Column("table", "molecule_10"));
		table.getColumnList().add(new Column("table", "molecule_11"));
		table.getColumnList().add(new Column("table", "molecule_12"));
		table.getColumnList().add(new Column("table", "HO"));
		table.getColumnList().add(new Column("table", "BO"));
	}

	@Test
	public void doppelganger() {
		assertTrue(isDoppelganger(table, new Column("table", "atom_id1")));
		assertTrue(isDoppelganger(table, new Column("table", "atom_id2")));
		assertTrue(isDoppelganger(table, new Column("table", "chair_id")));
		assertTrue(isDoppelganger(table, new Column("table", "chair_id2")));
		assertTrue(isDoppelganger(table, new Column("table", "busIdA")));
		assertTrue(isDoppelganger(table, new Column("table", "busIdB")));
		assertTrue(isDoppelganger(table, new Column("table", "cop1_id")));
		assertTrue(isDoppelganger(table, new Column("table", "cop2_id")));
		assertTrue(isDoppelganger(table, new Column("table", "AddressLine1")));
		assertTrue(isDoppelganger(table, new Column("table", "AddressLine2")));
		assertTrue(isDoppelganger(table, new Column("table", "fish")));
		assertTrue(isDoppelganger(table, new Column("table", "fish_2")));
		assertFalse(isDoppelganger(table, new Column("table", "unique")));
		assertFalse(isDoppelganger(table, new Column("table", "kid")));
		assertFalse(isDoppelganger(table, new Column("table", "kids")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_1")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_2")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_3")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_4")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_5")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_6")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_7")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_8")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_9")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_10")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_11")));
		assertFalse(isDoppelganger(table, new Column("table", "molecule_12")));
		assertFalse(isDoppelganger(table, new Column("table", "HO")));
		assertFalse(isDoppelganger(table, new Column("table", "BO")));
	}
}


<|start_filename|>src/org/simmetrics/tokenizers/Tokenizer.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


package org.simmetrics.tokenizers;

import java.util.List;
import java.util.Set;

import com.google.common.collect.Multiset;

/**
 * A tokenizer divides an input string into tokens. A tokenizer may not provide
 * a collection containing null.
 * <p>
 * It is recommend that implementations extend {@link AbstractTokenizer}
 * rather then implement this interface.
 */
public interface Tokenizer {

	/**
	 * Return tokenized version of a string as a list of tokens.
	 *
	 * @param input
	 *            input string to tokenize
	 *
	 * @return List tokenized version of a string
	 */
	public List<String> tokenizeToList(String input);

	/**
	 * Return tokenized version of a string as a multiset of tokens.
	 *
	 * @param input
	 *            input string to tokenize
	 * @return tokenized version of a string as a multiset
	 */
	public Multiset<String> tokenizeToMultiset(String input);
	
	/**
	 * Return tokenized version of a string as a set of tokens.
	 *
	 * @param input
	 *            input string to tokenize
	 * @return tokenized version of a string as a set
	 */
	public Set<String> tokenizeToSet(String input);

}


<|start_filename|>src/export/TGF.java<|end_filename|>
package export;

import main.CompoundRelationship;
import main.Table;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;

// Export into .tgf (trivial graph format, which can be opened with yEd and other graph layouting applications).
// Note that the edges are directed -> you can change their appearance to arrows.
// The format is described at:
//  https://en.wikipedia.org/wiki/Trivial_Graph_Format
public class TGF {
	public static void writeGraph(File file, List<Table> tables, List<CompoundRelationship> relationships) throws FileNotFoundException, UnsupportedEncodingException {

		// Create a unique integer identifier for each table
		SortedMap<String, Integer> tableMap = new TreeMap<>();
		for (int i = 0; i < tables.size(); i++) {
			tableMap.put(tables.get(i).getName(), i+1);
		}

		// Rewrite the old file
		try (PrintWriter writer = new PrintWriter(file, "UTF-8")) {
			// Print nodes
			for (Map.Entry<String, Integer> entry : tableMap.entrySet()) {
				writer.println(entry.getValue() + " " + entry.getKey());
			}

			// Print separator
			writer.println("#");

			// Print edges
			for (CompoundRelationship relationship : relationships) {
				writer.println(tableMap.get(relationship.getFkTable()) + " " + tableMap.get(relationship.getPkTable()));
			}
		}
	}

	public static void writeGraph(List<Table> tables, List<CompoundRelationship> relationships) throws FileNotFoundException, UnsupportedEncodingException {
		File file = new File("graph.tgf");
		writeGraph(file, tables, relationships);
	}
}


<|start_filename|>src/org/simmetrics/tokenizers/AbstractTokenizer.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

package org.simmetrics.tokenizers;

import java.util.HashSet;
import java.util.Set;

import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;

/**
 * Convenience tokenizer. Provides default implementation to tokenize to set and
 * multiset by calling {@link Tokenizer#tokenizeToList(String)}.
 */
public abstract class AbstractTokenizer implements Tokenizer {

	@Override
	public Set<String> tokenizeToSet(final String input) {
		return new HashSet<>(tokenizeToList(input));
	}

	@Override
	public Multiset<String> tokenizeToMultiset(final String input) {
		return HashMultiset.create(tokenizeToList(input));
	}

}


<|start_filename|>test/utility/LCSTest.java<|end_filename|>
package utility;

import org.junit.Test;
import utility.LCS;

import static org.junit.Assert.assertEquals;

public class LCSTest {

	@Test
	public void lcs() {
		assertEquals(4, LCS.getLongestCommonSubsequence("abcde", "bcdef"));
		assertEquals(5, LCS.getLongestCommonSubsequence("a bcd e", "x bcd x"));
		assertEquals(3, LCS.getLongestCommonSubsequence("bcd", "abcd"));
	}

}


<|start_filename|>src/export/DDL.java<|end_filename|>
package export;

import main.Column;
import main.CompoundRelationship;
import main.Table;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.List;


// Export into SQL data definition language (DDL), which can be loaded with Oracle SQL Data Modeler.
// The export is using Oracle syntax (e.g. the quotes are "",...)
public class DDL {
	public static void writeGraph(File file, List<Table> tables, List<CompoundRelationship> relationships) throws FileNotFoundException, UnsupportedEncodingException {

		// Rewrite the old file
		try (PrintWriter writer = new PrintWriter(file, "UTF-8")) {
			// Print nodes
			for (Table table : tables) {
				writer.println(getNode(table));
			}

			// Print edges
			writer.println(SQL.getAllFkQuery(relationships, '"', '"'));
		}
	}

	public static void writeGraph(List<Table> tables, List<CompoundRelationship> relationships) throws FileNotFoundException, UnsupportedEncodingException {
		File file = new File("graph.ddl");
		writeGraph(file, tables, relationships);
	}

	private static String getNode(Table table) {
		String result = "CREATE TABLE " + table.getName() + " ("+ System.lineSeparator();
		for (Column column : table.getColumnList()) {
			String pk = column.isPrimaryKey() ? " PRIMARY KEY" : "";
			result += "\t\"" + column.getName() + "\" " + column.getDataTypeName() + pk + "," + System.lineSeparator();
		}
		result = result.substring(0, result.length()-2);
		result += System.lineSeparator() + ");" + System.lineSeparator();
		return result;
	}
}


<|start_filename|>src/org/simmetrics/metrics/Math.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */


package org.simmetrics.metrics;

import java.util.Set;

import com.google.common.collect.Multiset;
import com.google.common.collect.Multisets;
import com.google.common.collect.Sets;

final class Math {

	private Math() {
		// Utility class
	}
	static float max(float a, float b, float c) {
		return java.lang.Math.max(java.lang.Math.max(a, b), c);
	}

	static int max(final int a, final int b, final int c) {
		return java.lang.Math.max(java.lang.Math.max(a, b), c);
	}

	static float max(float a, float b, float c, float d) {
		return java.lang.Math.max(
				java.lang.Math.max(java.lang.Math.max(a, b), c), d);
	}


	static int max(final int a, final int b, final int c, final int d) {
		return java.lang.Math.max(
				java.lang.Math.max(java.lang.Math.max(a, b), c), d);
	}

	static float min(float a, float b, float c) {
		return java.lang.Math.min(java.lang.Math.min(a, b), c);
	}

	static int min(final int a, final int b, final int c) {
		return java.lang.Math.min(java.lang.Math.min(a, b), c);
	}
	
	static float min(final float a, final float b, final float c,
			final float d) {
		return java.lang.Math.min(
				java.lang.Math.min(java.lang.Math.min(a, b), c), d);
	}

	static int min(final int a, final int b, final int c, final int d) {
		return java.lang.Math.min(
				java.lang.Math.min(java.lang.Math.min(a, b), c), d);
	}

	static <T> Multiset<T> union(Multiset<T> a, Multiset<T> b) {
		// Lager set first for performance improvement.
		// See: MathCaliper
		if (a.size() < b.size()) {
			return Multisets.union(b, a);
		}

		return Multisets.union(a, b);
	}

	static <T> Multiset<T> intersection(Multiset<T> a, Multiset<T> b) {
		// Smaller set first for performance improvement.
		// See: MathCaliper
		if (a.size() < b.size()) {
			return Multisets.intersection(a, b);
		}

		return Multisets.intersection(b, a);
	}

	static <T> Set<T> intersection(Set<T> a, Set<T> b) {
		// Smaller set first for performance improvement.
		// See: MathCaliper and note at Sets.intersection
		if (a.size() < b.size()) {
			return Sets.intersection(a, b);
		}

		return Sets.intersection(b, a);
	}
}


<|start_filename|>src/utility/Tokenization.java<|end_filename|>
package utility;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;

// See:
//  http://codereview.stackexchange.com/questions/62500/split-camel-cased-snake-cased-string
//  http://stackoverflow.com/questions/2559759/how-do-i-convert-camelcase-into-human-readable-names-in-java
public class Tokenization {
	private static final Pattern PUNCTSPACE = Pattern.compile("[ \\p{Punct}]+");
	private static final Pattern TRANSITION = Pattern.compile(
			String.format("%s|%s|%s",
					"(?<=[\\p{javaUpperCase}])(?=[\\p{javaUpperCase}][\\p{javaLowerCase}])",
					"(?<=[^\\p{javaUpperCase}])(?=[\\p{javaUpperCase}])",
					"(?<=[\\p{javaLowerCase}\\p{javaUpperCase}}])(?=[^\\p{javaLowerCase}\\p{javaUpperCase}])"
			)
	);

	public static List<String> split(String text) {
		List<String> result = new ArrayList<>();
		for (String word : PUNCTSPACE.split(text)) {
			if (word.isEmpty()) {
				continue;
			}
			Collections.addAll(result, TRANSITION.split(word));
		}
		return result.isEmpty() ? Collections.singletonList(text) : result;
	}

	public static List<String> lowercaseSplit(String text) {
		List<String> result = new ArrayList<>();
		for (String token : split(text)) {
			result.add(token.toLowerCase());
		}
		return result;
	}

	public static Boolean contains(List<String> lowerCaseTokens, List<String> keywords) {
		for (String keyword : keywords) {
			for (String token : lowerCaseTokens) {
				if (token.equals(keyword)) return true;
			}
		}
		return false;
	}
}


<|start_filename|>test/main/NullsExpectedTest.java<|end_filename|>
package main;

import org.junit.Assert;
import org.junit.Test;

import java.util.ArrayList;
import java.util.List;

public class NullsExpectedTest {
    @Test
    public void containsKeyword_insensitiveToPrefixesAndSuffixes() throws Exception {
        List<String> tokens = new ArrayList<>();

        tokens.add("apple");
        Assert.assertFalse(NullsExpected.containsKeyword(tokens));

        tokens.add("banana");
        Assert.assertFalse(NullsExpected.containsKeyword(tokens));

        tokens.add("enclosed");
        Assert.assertTrue(NullsExpected.containsKeyword(tokens)); // Because of "close" keyword
    }

}

<|start_filename|>src/main/Accuracy.java<|end_filename|>
package main;

import java.util.List;

public class Accuracy {

	public static double getPkPrecision(List<Table> tables) {
		double tp = 0.0;
		double fp = 0.0;

		for (Table table : tables) {
			for (Column column : table.getColumnList()) {
				if (column.isPrimaryKey() && column.isEstimatedPk()) tp++;
				if (!column.isPrimaryKey() && column.isEstimatedPk()) fp++;
			}
		}

		return tp/(tp+fp);
	}

	public static double getPkRecall(List<Table> tables) {
		double tp = 0.0;
		double fn = 0.0;

		for (Table table : tables) {
			for (Column column : table.getColumnList()) {
				if (column.isPrimaryKey() && column.isEstimatedPk()) tp++;
				if (column.isPrimaryKey() && !column.isEstimatedPk()) fn++;
			}
		}

		return tp/(tp+fn);
	}

	public static double getFkPrecision(List<Relationship> relationships) {
		double tp = 0.0;
		double fp = 0.0;

		for (Relationship relationship : relationships) {
			if (relationship.isForeignKey() && relationship.isEstimatedFk()) tp++;
			if (!relationship.isForeignKey() && relationship.isEstimatedFk()) fp++;
		}

		return tp/(tp+fp);
	}

	public static double getFkRecall(List<Relationship> relationships) {
		double tp = 0.0;
		double fn = 0.0;

		for (Relationship relationship : relationships) {
			if (relationship.isForeignKey() && relationship.isEstimatedFk()) tp++;
			if (relationship.isForeignKey() && !relationship.isEstimatedFk()) fn++;
		}

		return tp/(tp+fn);
	}
}


<|start_filename|>test/main/OptimizationTest.java<|end_filename|>
package main;

import org.junit.Assert;
import org.junit.Test;

import java.util.Stack;

public class OptimizationTest {

	// True: 2->1->3
	// False: 1->2, 1->1
	@Test
	public void isAcyclic() {

		Relationship relationshipTrue = new Relationship();
		relationshipTrue.setFkTable(new Table("schema", "tab2"));
		relationshipTrue.setFk(new Column("tab2", "col2"));
		relationshipTrue.setPkTable(new Table("schema", "tab1"));
		relationshipTrue.setPk(new Column("tab1", "col1"));
		relationshipTrue.setForeignKeyProbability(0.9);

		Relationship relationship2 = new Relationship();
		relationship2.setFkTable(new Table("schema", "tab1"));
		relationship2.setFk(new Column("tab1", "col1"));
		relationship2.setPkTable(new Table("schema", "tab3"));
		relationship2.setPk(new Column("tab3", "col1"));
		relationship2.setForeignKeyProbability(0.95);

		Relationship relationshipFalse = new Relationship();
		relationshipFalse.setFkTable(new Table("schema", "tab1"));
		relationshipFalse.setFk(new Column("tab1", "col1"));
		relationshipFalse.setPkTable(new Table("schema", "tab2"));
		relationshipFalse.setPk(new Column("tab2", "col2"));
		relationshipFalse.setForeignKeyProbability(0.4);

		Relationship relationshipSelf = new Relationship();
		relationshipSelf.setFkTable(new Table("schema", "tab1"));
		relationshipSelf.setFk(new Column("tab1", "col1"));
		relationshipSelf.setPkTable(new Table("schema", "tab1"));
		relationshipSelf.setPk(new Column("tab1", "col1"));
		relationshipSelf.setForeignKeyProbability(0.8);


		// Single acyclic
		Stack<Relationship> single = new Stack<>();
		single.add(relationshipTrue);
		Assert.assertTrue(OptimizationRelationship.isAcyclic(single));

		// Single cyclic
		Stack<Relationship> singleCyclic = new Stack<>();
		singleCyclic.add(relationshipSelf);
		Assert.assertFalse(OptimizationRelationship.isAcyclic(singleCyclic));

		// Cyclic
		Stack<Relationship> cyclic = new Stack<>();
		cyclic.add(relationshipTrue);
		cyclic.add(relationshipFalse);
		Assert.assertFalse(OptimizationRelationship.isAcyclic(cyclic));

		// Acyclic
		Stack<Relationship> acyclic = new Stack<>();
		acyclic.add(relationshipTrue);
		acyclic.add(relationship2);
		Assert.assertTrue(OptimizationRelationship.isAcyclic(acyclic));

		// Cyclic reverse
		Stack<Relationship> cyclicReverse = new Stack<>();
		cyclicReverse.add(relationshipFalse);
		cyclicReverse.add(relationshipTrue);
		Assert.assertFalse(OptimizationRelationship.isAcyclic(cyclicReverse));

		// Cyclic with blind ends (still, the last relationship must be a part of the cycle to make the cycle detectable)
		Stack<Relationship> cyclicMisleading = new Stack<>();
		cyclicMisleading.add(relationship2);
		cyclicMisleading.add(relationshipFalse);
		cyclicMisleading.add(relationship2);
		cyclicMisleading.add(relationshipTrue);
		Assert.assertFalse(OptimizationRelationship.isAcyclic(cyclicMisleading));
	}

	@Test
	public void piecewise() {
		OptimizationRelationship.lowerEstimate = 3;
		OptimizationRelationship.expectedCount = 5;
		OptimizationRelationship.upperEstimate = 7;

		for (int i = 0; i < 10; i++) {
			System.out.println(i + "  " + -OptimizationRelationship.piecewise(i) + "  " + (OptimizationRelationship.piecewise(i)- OptimizationRelationship.piecewise(i+1)) );
		}
	}

	@Test
	public void piecewise2() {
		OptimizationRelationship.lowerEstimate = 3.5;
		OptimizationRelationship.expectedCount = 5.5;
		OptimizationRelationship.upperEstimate = 7.5;

		for (int i = 0; i < 10; i++) {
			System.out.println(i + "  " + -OptimizationRelationship.piecewise(i) + "  " + (OptimizationRelationship.piecewise(i)- OptimizationRelationship.piecewise(i+1)) );
		}
	}
}


<|start_filename|>src/vendor/Default.java<|end_filename|>
package vendor;

import main.Table;

import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;

// A fallback. If Linkifier is connected to some unsupported database, leave the statistics unfilled/filled with the defaults.
public class Default implements Vendor {

	public void getTableStatistics(String databaseName, String schemaName, List<Table> tables, Connection connection) throws SQLException {

	}

	public void getColumnStatistics(String databaseName, String schemaName, List<Table> tables, Connection connection) throws SQLException {

	}
}


<|start_filename|>src/main/Doppelganger.java<|end_filename|>
package main;

import java.util.ArrayList;
import java.util.List;

public class Doppelganger {

	// Identify doppelganger columns like {atom_id1 & atom_id2} or {atom_id & atom_id2} or {atomIdA & atomIdB} or {atom1_id & atom2_id} or {atom & atom_2}.
	// We do not identify columns like {A1, A2, A3, A4,..., A12} as doppelgangers as they are likely a part of an un-normalized table.
	// And as such they are not part of a foreign key constraint.
	// If too many columns in the table are doppelgangers, we should return false.
	public static boolean isDoppelganger(Table table, Column column) {
		String a = column.getName();
		boolean hasDoppelganger;
		List<Column> candidates = new ArrayList<>();

		for (Column column2 : table.getColumnList()) {
			String b = column2.getName();
			boolean diffContainsGood = false;
			boolean diffContainsBad = false;
			int diffCount = 0;
			int diffAscii = 0;
			int numberCount = 0;
			int lengthDifference = Math.abs(a.length() - b.length());
			int length = Math.min(a.length(), b.length());

			// Collect statistics
			for (int i = 0; i < length; i++) {
				numberCount += (Character.isDigit(a.charAt(i)) ? 1:0);
				if (a.charAt(i) != b.charAt(i)) {
					diffCount++;
					diffAscii += Math.abs(a.charAt(i)-b.charAt(i));
					diffContainsGood = (a.charAt(i) == 'b' || a.charAt(i) == 'B' || a.charAt(i) == '2' || b.charAt(i) == 'b' || b.charAt(i) == 'B' || b.charAt(i) == '2');
					diffContainsBad = (a.charAt(i) == 'c' || a.charAt(i) == 'C' || a.charAt(i) == '3' || b.charAt(i) == 'c' || b.charAt(i) == 'C' || b.charAt(i) == '3');
				}
			}

			// It is likely a column of a denormalized table. We can immediately terminate the search.
			if (numberCount>1) return false;
			if (lengthDifference == 0 && diffCount == 1 && diffContainsBad) return false;

			// It could be a doppelganger.
			if (lengthDifference == 0 && diffCount == 1 && diffAscii == 1 && diffContainsGood) candidates.add(column2);
			if (lengthDifference == 1 && diffCount == 0) {
				if (a.length() > b.length()) {
					hasDoppelganger = (a.charAt(length) == 'b' || a.charAt(length) == 'B' || a.charAt(length) == '2');
				} else {
					hasDoppelganger = (b.charAt(length) == 'b' || b.charAt(length) == 'B' || b.charAt(length) == '2');
				}
				if (hasDoppelganger) candidates.add(column2);
			}
			if (lengthDifference == 2 && diffCount == 0) {
				hasDoppelganger = false;
				if (a.length() > b.length() && a.charAt(length)=='_') {
					hasDoppelganger = (a.charAt(length+1) == 'b' || a.charAt(length+1) == 'B' || a.charAt(length+1) == '2');
				} else if (b.length() > a.length() && b.charAt(length)=='_') {
					hasDoppelganger = (b.charAt(length+1) == 'b' || b.charAt(length+1) == 'B' || b.charAt(length+1) == '2');
				}
				if (hasDoppelganger) candidates.add(column2);
			}
		}

		if (candidates.size()==1) column.setDoppelgangerName(candidates.get(0).getName());

		// Asses the count of the candidates
		return (candidates.size()==1);
	}

}


<|start_filename|>src/org/simmetrics/metrics/TanimotoCoefficient.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 * #L%
 */

package org.simmetrics.metrics;

import static org.simmetrics.metrics.Math.intersection;
import static java.lang.Math.sqrt;

import java.util.Set;

import org.simmetrics.SetMetric;

/**
 * Calculates the Tanimoto similarity coefficient over two sets. The
 * similarity is defined as the cosine of the angle between the sets
 * expressed as sparse vectors.
 * <p>
 * <code>
 * similarity(a,b) = a·b / (||a|| * ||b||)
 * </code>
 * <p>
 * The cosine similarity is identical to the Tanimoto coefficient, but unlike
 * Tanimoto the occurrence (cardinality) of an entry is taken into account. E.g.
 * {@code [hello, world]} and {@code [hello, world, hello, world]} would be
 * identical when compared with Tanimoto but are dissimilar when the cosine
 * similarity is used.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see CosineSimilarity
 * @see <a href="http://en.wikipedia.org/wiki/Cosine_similarity">Wikipedia
 *      Cosine similarity</a>
 * 
 * @param <T>
 *            type of the token
 */
public final class TanimotoCoefficient<T> implements SetMetric<T> {

	@Override
	public float compare(Set<T> a, Set<T> b) {

		if (a.isEmpty() && b.isEmpty()) {
			return 1.0f;
		}

		if (a.isEmpty() || b.isEmpty()) {
			return 0.0f;
		}

		// a·b / (||a|| * ||b||)
		// Dot product of two binary vectors is the intersection of two sets
		// Magnitude of a binary vectors is square root of its size.
		return (float) (intersection(a, b).size() / (sqrt(a.size()) * sqrt(b.size())));
	}

	@Override
	public String toString() {
		return "TanimotoCoefficient";
	}

}


<|start_filename|>test/main/CompoundRelationshipTest.java<|end_filename|>
package main;

import export.SQL;
import org.junit.Test;

import java.util.ArrayList;
import java.util.List;

import static org.junit.Assert.assertEquals;

public class CompoundRelationshipTest {

	@Test
	public void simpleFkTest() {
		List<Relationship> relationships = new ArrayList<>();

		Table t1 = new Table("schema", "t1");
		Column c1 = new Column("t1", "c1", t1);
		c1.setBestAttemptPk(true);
		t1.getColumnList().add(c1);

		Table t2 = new Table("schema", "t2");
		Column c2 = new Column("t2", "c1", t2);
		c2.setBestAttemptPk(true);
		t2.getColumnList().add(c2);
		Column c3 = new Column("t2", "c3", t2);
		t2.getColumnList().add(c3);

		Relationship r1 = new Relationship();
		r1.setFkTable(t1);
		r1.setPkTable(t2);
		r1.setFk(c1);
		r1.setPk(c2);
		r1.setForeignKeyProbability(0.98);
		r1.setEstimatedFk(true);
		relationships.add(r1);

		// Run
		List<CompoundRelationship> compoundRelationships = CompoundRelationship.buildFrom(relationships);
		String actual = SQL.getFkQuery(compoundRelationships, '`', '`');
		String expected = "ALTER TABLE `t1` ADD FOREIGN KEY (`c1`) REFERENCES `t2`(`c1`);";
		assertEquals(expected.trim(), actual.trim());
	}

	@Test
	public void compoundFkTest() {
		List<Relationship> relationships = new ArrayList<>();

		Table t3 = new Table("schema", "t3");
		Column c4 = new Column("t3", "c1", t3);
		c4.setBestAttemptPk(true);
		t3.getColumnList().add(c4);
		Column c5 = new Column("t3", "c3", t3);
		c5.setBestAttemptPk(true);
		t3.getColumnList().add(c5);

		Table t4 = new Table("schema", "t4");
		Column c6 = new Column("t4", "c1", t4);
		c6.setBestAttemptPk(true);
		t4.getColumnList().add(c4);
		Column c7 = new Column("t4", "c3", t4);
		c7.setBestAttemptPk(true);
		t4.getColumnList().add(c7);

		Relationship r2 = new Relationship();
		r2.setFkTable(t3);
		r2.setPkTable(t4);
		r2.setFk(c4);
		r2.setPk(c6);
		r2.setForeignKeyProbability(0.98);
		r2.setEstimatedFk(true);
		relationships.add(r2);

		Relationship r3 = new Relationship();
		r3.setFkTable(t3);
		r3.setPkTable(t4);
		r3.setFk(c5);
		r3.setPk(c7);
		r3.setForeignKeyProbability(0.98);
		r3.setEstimatedFk(true);
		relationships.add(r3);

		// Run
		List<CompoundRelationship> compoundRelationships = CompoundRelationship.buildFrom(relationships);
		String actual = SQL.getFkQuery(compoundRelationships, '`', '`');
		String expected = "ALTER TABLE `t3` ADD FOREIGN KEY (`c1`, `c3`) REFERENCES `t4`(`c1`, `c3`);";
		assertEquals(expected.trim(), actual.trim());
	}



}


<|start_filename|>src/org/simmetrics/simplifiers/ColognePhonetic.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */
package org.simmetrics.simplifiers;

import static com.google.common.base.Preconditions.checkNotNull;

/**
 * Encodes a string into a Cologne Phonetic value.
 * <p>
 * Implements the <a
 * href="http://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik">K&ouml;lner
 * Phonetik</a> (Cologne Phonetic) algorithm issued by <NAME> in
 * 1969.
 * <p>
 * The <i>K&ouml;lner Phonetik</i> is a phonetic algorithm which is optimized
 * for the German language. It is related to the well-known soundex algorithm.
 * <p>
 * This class is immutable and thread-safe.
 * 
 * @see org.apache.commons.codec.language.ColognePhonetic
 * 
 * @deprecated will be removed due to a lack of a good use case
 */
@Deprecated
public final class ColognePhonetic implements Simplifier {

	private final org.apache.commons.codec.language.ColognePhonetic simplifier = new org.apache.commons.codec.language.ColognePhonetic();

	@Override
	public String simplify(String input) {
		checkNotNull(input);
		return simplifier.encode(input);
	}

	@Override
	public String toString() {
		return "ColognePhonetic";
	}
	
	

}


<|start_filename|>test/utility/MultiSetTest.java<|end_filename|>
package utility;

import org.junit.Test;

import static org.junit.Assert.assertEquals;

public class MultiSetTest {

	@Test
	public void batteryTest() {
		MultiSet<String> multiSet = new MultiSet<>();

		assertEquals(0, multiSet.totalCount());
		assertEquals(0, multiSet.count("a"));
		multiSet.add("a");
		assertEquals(1, multiSet.count("a"));
		multiSet.add("a");
		assertEquals(2, multiSet.count("a"));
		multiSet.add("b");
		assertEquals(1, multiSet.count("b"));
		assertEquals(2, multiSet.count("a"));
		multiSet.add("a");
		assertEquals(3, multiSet.count("a"));
		assertEquals(4, multiSet.totalCount());
	}

}


<|start_filename|>src/utility/LCS.java<|end_filename|>
package utility;

public class LCS {

	// Source: http://www.programcreek.com/2014/04/longest-common-subsequence-java/
	// LCS is significantly slower than Levenshtein or Jaro-Winkler -> use LCS only if necessary.
	public static int getLongestCommonSubsequence(String a, String b){
		int m = a.length();
		int n = b.length();
		int[][] dp = new int[m+1][n+1];

		for(int i=0; i<=m; i++){
			for(int j=0; j<=n; j++){
				if(i==0 || j==0){
					dp[i][j]=0;
				}else if(a.charAt(i-1)==b.charAt(j-1)){
					dp[i][j] = 1 + dp[i-1][j-1];
				}else{
					dp[i][j] = Math.max(dp[i-1][j], dp[i][j-1]);
				}
			}
		}

		return dp[m][n];
	}

}

<|start_filename|>src/org/simmetrics/builders/StringDistanceBuilder.java<|end_filename|>
/*
 * #%L
 * Simmetrics Core
 * %%
 * Copyright (C) 2014 - 2016 Simmetrics Authors
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

package org.simmetrics.builders;

import static com.google.common.base.Preconditions.checkNotNull;
import static org.simmetrics.builders.StringDistances.create;
import static org.simmetrics.builders.StringDistances.createForListDistance;
import static org.simmetrics.builders.StringDistances.createForMultisetDistance;
import static org.simmetrics.builders.StringDistances.createForSetDistance;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;

import org.simmetrics.ListDistance;
import org.simmetrics.Distance;
import org.simmetrics.MultisetDistance;
import org.simmetrics.SetDistance;
import org.simmetrics.StringDistance;
import org.simmetrics.simplifiers.Simplifier;
import org.simmetrics.simplifiers.Simplifiers;
import org.simmetrics.tokenizers.Tokenizer;
import org.simmetrics.tokenizers.Tokenizers;

import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.cache.Cache;
import com.google.common.collect.Multiset;

/**
 * Convenience tool to build string distance metrics. Any class implementing
 * {@link StringDistance}, {@link ListDistance}, {@link SetDistance} or
 * {@link MultisetDistance} can be used to build a string distance metric.
 * Supports the addition of simplification, tokenization, token-filtering,
 * token-transformation and caching to a distance.
 * <p>
 * The created distance metrics are immutable and thread-safe provided all their components
 * are also immutable and thread-safe.
 * <p>
 * For usage examples see the simmetrics-example module.
 */
public final class StringDistanceBuilder {

	private StringDistanceBuilder() {
		// Utility class
	}

	/**
	 * Starts building a distance metric with a string distance metric.
	 * 
	 * @param distance
	 *            the distance to use as a base
	 * @return a builder for fluent chaining
	 */
	public static StringDistanceInitialSimplifierStep with(StringDistance distance) {
		return new CompositeStringDistanceBuilder(distance);
	}

	/**
	 * Starts building a distance with a list distance.
	 * 
	 * @param distance
	 *            the distance to use as a base
	 * @return a builder for fluent chaining
	 */
	public static CollectionDistanceInitialSimplifierStep<List<String>> with(ListDistance<String> distance) {
		return new CompositeListDistanceBuilder(distance);

	}

	/**
	 * Starts building a distance with a set distance.
	 * 
	 * @param distance
	 *            the distance to use as a base
	 * @return a builder for fluent chaining
	 */
	public static CollectionDistanceInitialSimplifierStep<Set<String>> with(SetDistance<String> distance) {
		return new CompositeSetDistanceBuilder(distance);

	}

	/**
	 * Starts building a distance with a multiset distance.
	 * 
	 * @param distance
	 *            the distance to use as a base
	 * @return a builder for fluent chaining
	 */
	public static CollectionDistanceInitialSimplifierStep<Multiset<String>> with(MultisetDistance<String> distance) {
		return new CompositeMultisetDistanceBuilder(distance);

	}

	@SuppressWarnings("javadoc")
	public interface BuildStep {
		/**
		 * Builds a distance with the given steps.
		 * 
		 * @return a distance
		 */
		StringDistance build();

	}

	@SuppressWarnings("javadoc")
	public interface StringDistanceInitialSimplifierStep extends BuildStep {
		/**
		 * Adds a simplifier to the distance.
		 * 
		 * @param simplifier
		 *            a simplifier to add
		 * @return this for fluent chaining
		 */
		StringDistanceSimplifierStep simplify(Simplifier simplifier);

		/**
		 * Builds a distance with the given simplifier.
		 * 
		 * @return a distance
		 */
		@Override
		StringDistance build();

	}

	@SuppressWarnings("javadoc")
	public interface StringDistanceSimplifierStep extends StringDistanceInitialSimplifierStep {
		/**
		 * Adds a simplifier to the distance.
		 * 
		 * @param simplifier
		 *            a simplifier to add
		 * @return this for fluent chaining
		 */
		@Override
		StringDistanceSimplifierStep simplify(Simplifier simplifier);

		/**
		 * Sets a cache for simplification chain. The cache will store the
		 * result of all previous simplification steps.
		 * 
		 * @param cache
		 *            a cache to add
		 * @return this for fluent chaining
		 */
		BuildStep cacheStrings(Cache<String, String> cache);

		/**
		 * Builds a distance with the given simplifier.
		 * 
		 * @return a distance
		 */
		@Override
		StringDistance build();

	}

	@SuppressWarnings("javadoc")
	public interface CollectionDistanceInitialSimplifierStep<T extends Collection<String>> {
		/**
		 * Adds a simplifier to the distance.
		 * 
		 * @param simplifier
		 *            a simplifier to add
		 * @return this for fluent chaining
		 */
		CollectionDistanceSimplifierStep<T> simplify(Simplifier simplifier);

		/**
		 * Adds a tokenization step to the distance.
		 * 
		 * @param tokenizer
		 *            a tokenizer to add
		 * @return this for fluent chaining
		 */
		CollectionDistanceTokenizerStep<T> tokenize(Tokenizer tokenizer);

	}

	@SuppressWarnings("javadoc")
	public interface CollectionDistanceSimplifierStep<T extends Collection<String>>
			extends CollectionDistanceInitialSimplifierStep<T> {
		/**
		 * Adds a simplifier to the distance.
		 * 
		 * @param simplifier
		 *            a simplifier to add
		 * @return this for fluent chaining
		 */
		@Override
		CollectionDistanceSimplifierStep<T> simplify(Simplifier simplifier);

		/**
		 * Sets a cache for simplification chain. The cache will store the
		 * result of all previous simplification steps.
		 * 
		 * @param cache
		 *            a cache to add
		 * @return this for fluent chaining
		 */
		CollectionDistanceInitialTokenizerStep<T> cacheStrings(Cache<String, String> cache);

		/**
		 * Adds a tokenization step to the distance.
		 * 
		 * @param tokenizer
		 *            a tokenizer to add
		 * @return this for fluent chaining
		 */
		@Override
		CollectionDistanceTokenizerStep<T> tokenize(Tokenizer tokenizer);

	}

	@SuppressWarnings("javadoc")
	public interface CollectionDistanceInitialTokenizerStep<T extends Collection<String>> {
		/**
		 * Adds a tokenization step to the distance.
		 * 
		 * @param tokenizer
		 *            a tokenizer to add
		 * @return a builder for fluent chaining
		 */
		CollectionDistanceTokenizerStep<T> tokenize(Tokenizer tokenizer);

	}

	@SuppressWarnings("javadoc")
	public interface CollectionDistanceTokenizerStep<T extends Collection<String>>
			extends BuildStep, CollectionDistanceInitialTokenizerStep<T> {
		/**
		 * Adds a tokenization step to the distance.
		 * 
		 * @param tokenizer
		 *            a tokenizer to add
		 * @return a builder for fluent chaining
		 */
		@Override
		CollectionDistanceTokenizerStep<T> tokenize(Tokenizer tokenizer);

		/**
		 * Adds a filter step to the distance. All tokens that match the
		 * predicate are kept.
		 * 
		 * @param predicate
		 *            a predicate for tokens to keep
		 * @return this for fluent chaining
		 */
		CollectionDistanceTokenizerStep<T> filter(Predicate<String> predicate);

		/**
		 * Adds a transform step to the distance. All tokens are transformed by
		 * the function. The function may not return null.
		 * 
		 * @param function
		 *            a function to transform tokens
		 * @return this for fluent chaining
		 */
		CollectionDistanceTokenizerStep<T> transform(Function<String, String> function);

		/**
		 * Sets a cache for tokenization chain. The cache will store the result
		 * of all previous tokenization steps.
		 * 
		 * @param cache
		 *            a cache to add
		 * @return this for fluent chaining
		 * 
		 */
		BuildStep cacheTokens(Cache<String, T> cache);

		/**
		 * Builds a string distance metric that will use the given
		 * simplification, tokenization and filtering steps.
		 * 
		 * @return a string distance metric
		 */
		@Override
		StringDistance build();

	}

	private static final class CompositeStringDistanceBuilder implements StringDistanceSimplifierStep {

		private final Distance<String> distance;

		private final List<Simplifier> simplifiers = new ArrayList<>();

		CompositeStringDistanceBuilder(Distance<String> distance) {
			checkNotNull(distance);
			this.distance = distance;
		}

		@Override
		public StringDistance build() {

			if (simplifiers.isEmpty()) {
				return create(distance);
			}
			return create(distance, chainSimplifiers());
		}

		private Simplifier chainSimplifiers() {
			final Simplifier simplifier = Simplifiers.chain(simplifiers);
			simplifiers.clear();

			return simplifier;
		}

		@Override
		public BuildStep cacheStrings(Cache<String, String> cache) {
			checkNotNull(cache);
			CachingSimplifier cachingSimplifier = new CachingSimplifier(cache, chainSimplifiers());
			this.simplifiers.add(cachingSimplifier);
			return this;
		}

		@Override
		public StringDistanceSimplifierStep simplify(Simplifier simplifier) {
			checkNotNull(simplifier);
			this.simplifiers.add(simplifier);
			return this;
		}

	}

	private static abstract class CompositeCollectionDistanceBuilder<T extends Collection<String>>
			implements CollectionDistanceSimplifierStep<T>, CollectionDistanceTokenizerStep<T> {

		private final Distance<T> distance;

		private final List<Simplifier> simplifiers = new ArrayList<>();
		private final List<Tokenizer> tokenizers = new ArrayList<>();

		CompositeCollectionDistanceBuilder(Distance<T> distance) {
			checkNotNull(distance);
			this.distance = distance;
		}

		@Override
		public final StringDistance build() {

			Tokenizer tokenizer = chainTokenizers();

			if (simplifiers.isEmpty()) {
				return build(distance, tokenizer);
			}

			return build(distance, chainSimplifiers(), tokenizer);
		}

		abstract StringDistance build(Distance<T> distance, Simplifier simplifier, Tokenizer tokenizer);

		abstract StringDistance build(Distance<T> distance, Tokenizer tokenizer);

		@Override
		public final BuildStep cacheTokens(Cache<String, T> cache) {
			checkNotNull(cache);
			tokenizers.add(createCachingTokenizer(cache, chainTokenizers()));
			return this;
		}

		protected abstract Tokenizer createCachingTokenizer(Cache<String, T> cache, Tokenizer tokenizer);

		@Override
		public final CollectionDistanceInitialTokenizerStep<T> cacheStrings(Cache<String, String> cache) {
			checkNotNull(cache);

			CachingSimplifier cachingSimplifier = new CachingSimplifier(cache, chainSimplifiers());
			this.simplifiers.add(cachingSimplifier);

			return this;
		}

		@Override
		public final CollectionDistanceSimplifierStep<T> simplify(Simplifier simplifier) {
			checkNotNull(simplifier);
			simplifiers.add(simplifier);
			return this;
		}

		@Override
		public final CollectionDistanceTokenizerStep<T> tokenize(Tokenizer tokenizer) {
			checkNotNull(tokenizer);
			tokenizers.add(tokenizer);
			return this;
		}

		@Override
		public final CollectionDistanceTokenizerStep<T> filter(Predicate<String> predicate) {
			checkNotNull(predicate);

			final Tokenizer filter = Tokenizers.filter(chainTokenizers(), predicate);

			tokenizers.add(filter);

			return this;
		}

		@Override
		public final CollectionDistanceTokenizerStep<T> transform(Function<String, String> function) {
			checkNotNull(function);
			final Tokenizer transform = Tokenizers.transform(chainTokenizers(), function);
			tokenizers.add(transform);

			return this;
		}

		private Tokenizer chainTokenizers() {
			final Tokenizer tokenizer = Tokenizers.chain(tokenizers);
			tokenizers.clear();
			return tokenizer;
		}

		private Simplifier chainSimplifiers() {
			final Simplifier simplifier = Simplifiers.chain(simplifiers);
			simplifiers.clear();
			return simplifier;
		}

	}

	private static final class CompositeListDistanceBuilder extends CompositeCollectionDistanceBuilder<List<String>> {

		CompositeListDistanceBuilder(Distance<List<String>> distance) {
			super(distance);
		}

		@Override
		StringDistance build(Distance<List<String>> distance, Simplifier simplifier, Tokenizer tokenizer) {
			return createForListDistance(distance, simplifier, tokenizer);
		}

		@Override
		StringDistance build(Distance<List<String>> distance, Tokenizer tokenizer) {
			return createForListDistance(distance, tokenizer);
		}

		@Override
		protected Tokenizer createCachingTokenizer(Cache<String, List<String>> cache, Tokenizer tokenizer) {
			return new CachingListTokenizer(cache, tokenizer);
		}

	}

	private static final class CompositeSetDistanceBuilder extends CompositeCollectionDistanceBuilder<Set<String>> {

		CompositeSetDistanceBuilder(Distance<Set<String>> distance) {
			super(distance);
		}

		@Override
		StringDistance build(Distance<Set<String>> distance, Simplifier simplifier, Tokenizer tokenizer) {
			return createForSetDistance(distance, simplifier, tokenizer);
		}

		@Override
		StringDistance build(Distance<Set<String>> distance, Tokenizer tokenizer) {
			return createForSetDistance(distance, tokenizer);
		}

		@Override
		protected Tokenizer createCachingTokenizer(Cache<String, Set<String>> cache, Tokenizer tokenizer) {
			return new CachingSetTokenizer(cache, tokenizer);
		}

	}

	private static final class CompositeMultisetDistanceBuilder
			extends CompositeCollectionDistanceBuilder<Multiset<String>> {

		CompositeMultisetDistanceBuilder(Distance<Multiset<String>> distance) {
			super(distance);
		}

		@Override
		StringDistance build(Distance<Multiset<String>> distance, Simplifier simplifier, Tokenizer tokenizer) {
			return createForMultisetDistance(distance, simplifier, tokenizer);
		}

		@Override
		StringDistance build(Distance<Multiset<String>> distance, Tokenizer tokenizer) {
			return createForMultisetDistance(distance, tokenizer);
		}

		@Override
		protected Tokenizer createCachingTokenizer(Cache<String, Multiset<String>> cache, Tokenizer tokenizer) {
			return new CachingMultisetTokenizer(cache, tokenizer);
		}

	}

	static final class CachingSimplifier implements Simplifier {

		private final Cache<String, String> cache;
		final Simplifier simplifier;

		CachingSimplifier(Cache<String, String> cache, Simplifier simplifier) {
			this.cache = cache;
			this.simplifier = simplifier;
		}

		@Override
		public String simplify(final String input) {
			try {
				return cache.get(input, new Callable<String>() {

					@Override
					public String call() throws Exception {
						return simplifier.simplify(input);
					}

				});
			} catch (ExecutionException e) {
				// Can't happen. Simplifier may not throw checked exceptions
				throw new IllegalStateException(e);
			}
		}

		@Override
		public String toString() {
			return "CachingSimplifier [" + simplifier + "]";
		}

	}

	static final class CachingMultisetTokenizer implements Tokenizer {

		private final Cache<String, Multiset<String>> cache;
		final Tokenizer tokenizer;

		CachingMultisetTokenizer(Cache<String, Multiset<String>> cache, Tokenizer tokenizer) {
			this.cache = cache;
			this.tokenizer = tokenizer;
		}

		@Override
		public List<String> tokenizeToList(final String input) {
			throw new UnsupportedOperationException();
		}

		@Override
		public Set<String> tokenizeToSet(final String input) {
			throw new UnsupportedOperationException();
		}

		@Override
		public Multiset<String> tokenizeToMultiset(final String input) {
			try {
				return cache.get(input, new Callable<Multiset<String>>() {

					@Override
					public Multiset<String> call() {
						return tokenizer.tokenizeToMultiset(input);
					}
				});
			} catch (ExecutionException e) {
				// Can't happen. Tokenizer may not throw checked exceptions
				throw new IllegalStateException(e);
			}
		}

		@Override
		public String toString() {
			return "CachingMultisetTokenizer [" + cache + ", " + tokenizer + "]";
		}
	}

	static final class CachingSetTokenizer implements Tokenizer {

		private final Cache<String, Set<String>> cache;
		final Tokenizer tokenizer;

		CachingSetTokenizer(Cache<String, Set<String>> cache, Tokenizer tokenizer) {
			this.cache = cache;
			this.tokenizer = tokenizer;
		}

		@Override
		public List<String> tokenizeToList(final String input) {
			throw new UnsupportedOperationException();
		}

		@Override
		public Set<String> tokenizeToSet(final String input) {
			try {
				return cache.get(input, new Callable<Set<String>>() {

					@Override
					public Set<String> call() {
						return tokenizer.tokenizeToSet(input);
					}
				});
			} catch (ExecutionException e) {
				// Can't happen. Tokenizer may not throw checked exceptions
				throw new IllegalStateException(e);
			}
		}

		@Override
		public Multiset<String> tokenizeToMultiset(final String input) {
			throw new UnsupportedOperationException();
		}

		@Override
		public String toString() {
			return "CachingSetTokenizer [" + cache + ", " + tokenizer + "]";
		}
	}

	static final class CachingListTokenizer implements Tokenizer {

		private final Cache<String, List<String>> cache;
		final Tokenizer tokenizer;

		CachingListTokenizer(Cache<String, List<String>> cache, Tokenizer tokenizer) {
			this.cache = cache;
			this.tokenizer = tokenizer;
		}

		@Override
		public List<String> tokenizeToList(final String input) {
			try {
				return cache.get(input, new Callable<List<String>>() {

					@Override
					public List<String> call() {
						return tokenizer.tokenizeToList(input);
					}
				});
			} catch (ExecutionException e) {
				// Can't happen. Tokenizer may not throw checked exceptions
				throw new IllegalStateException(e);
			}
		}

		@Override
		public Set<String> tokenizeToSet(final String input) {
			throw new UnsupportedOperationException();
		}

		@Override
		public Multiset<String> tokenizeToMultiset(final String input) {
			throw new UnsupportedOperationException();
		}

		@Override
		public String toString() {
			return "CachingListTokenizer [" + cache + ", " + tokenizer + "]";
		}
	}

}


<|start_filename|>test/utility/ChenTest.java<|end_filename|>
package utility;

import main.Column;
import main.Table;
import org.junit.Test;
import utility.Chen;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import static org.junit.Assert.assertEquals;

public class ChenTest {

	@Test
	public void batteryTest() {
		List<Table> tables = new ArrayList<>();

		Table t1 = new Table("schema", "t1");
		t1.setTokenizedLowerCaseTrimmedName(Arrays.asList("t", "1"));
		Column c1 = new Column("t1", "c1");
		c1.setTokenizedLowerCaseTrimmedName(Arrays.asList("c", "1"));
		t1.getColumnList().add(c1);
		tables.add(t1);

		Table t2 = new Table("schema", "t2");
		t2.setTokenizedLowerCaseTrimmedName(Arrays.asList("t", "2"));
		Column c2 = new Column("t2", "c1");
		c2.setTokenizedLowerCaseTrimmedName(Arrays.asList("c", "1"));
		t2.getColumnList().add(c2);
		Column c3 = new Column("t2", "c3");
		c3.setTokenizedLowerCaseTrimmedName(Arrays.asList("c", "3"));
		t2.getColumnList().add(c3);
		tables.add(t2);

		// Run
		Chen chen = new Chen(tables);
		assertEquals(1.0, chen.getChenSimilarity(c1, c2), 0.0001);
		assertEquals(0.5, chen.getChenSimilarity(c1, c3), 0.0001);
	}



}


<|start_filename|>src/export/CSV.java<|end_filename|>
package export;

import main.Relationship;
import main.Setting;
import main.Table;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.List;

public class CSV {

	public static void writePk(File file, List<Table> tables) throws FileNotFoundException, UnsupportedEncodingException {
		try (PrintWriter writer = new PrintWriter(file, "UTF-8")) {
			writer.println(Table.getHeader() + Setting.CSV_SEPARATOR + "primaryKeyProbability");
			for (Table table : tables) {
				writer.print(table.toProbability());
			}
		}
	}

	public static void writeFk(File file, List<Relationship> relationships) throws FileNotFoundException, UnsupportedEncodingException {
		try (PrintWriter writer = new PrintWriter(file, "UTF-8")) {
			writer.println(Relationship.getHeader() + Setting.CSV_SEPARATOR + "foreignKeyConstraintProbability");
			for (Relationship relationship : relationships) {
				writer.println(relationship.toProbability());
			}
		}
	}

	public static void writePk(List<Table> tables) throws FileNotFoundException, UnsupportedEncodingException {
		File file = new File("probability_pk.csv");
		writePk(file, tables);
	}

	public static void writeFk(List<Relationship> relationships) throws FileNotFoundException, UnsupportedEncodingException {
		File file = new File("probability_fk.csv");
		writeFk(file, relationships);
	}
}
