text:
\section{Detailed Description of \textit{Within-Dataset} Axioms}

\label{app:within}

\textbf{(Axiom W1) \textit{Scale Invariance.} } \textit{A measure $f$ satisfies scale invariance if $\forall X$, $ \forall d$, $\forall C$ over $(X, d)$, $f(C, X, d) = f(C, X, \alpha d)$ $\forall \alpha > 0$ (where  $\alpha d$ is a distance function satisfying $\alpha d(x,y) = \alpha \cdot d(x,y)$, $ \forall x, y \in X$) }

\textbf{$C$-consistent variant.} $d'$ is a $C$-consistent variant of $d$ if $d'(x, y) \leq d(x,y)$ implies $x \sim_C y$, and $d'(x, y) \geq d(x,y)$ implies $ x \nsim_C y$

\textbf{(Axiom W2) \textit{Consistency.} } \textit{A measure $f$ satisfies consistency if  $\forall X$, $ \forall d$, $\forall C$ over $(X, d)$, whenever $d'$ is a $C$-consistent variant of $d$, $f(C, X, d') \geq f(C, X, d)$.}

\textbf{(Axiom W3) \textit{Richness. }} \textit{A measure $f$ satisfies richness if $\forall X$, $\forall C$ over $X$, there exists $d$ over $X$ such that $C = \arg\max_{C_i \in \mathcal{C}}f(C_i, X, d)$ (where  $\mathcal{C}$ denotes the set of all possible clusterings over $X$)}

\textbf{Clustering Isomorphism. } \textit{Two clustering $C, C'$ over $(X, d)$ are isomorphic (i.e., $C \approx_d C' $), if there exists distance-preserving isomorphism $\phi: X \rightarrow X$, such that $\forall x, y \in X$, $x \sim_C y$ if and only if $\phi(x) \sim_{C'} \phi(y)$.}

\textbf{(Axiom W4) \textit{Isomorphism Invariance}.} \textit{A measure $f$ satisfies isomorphism invariance if $\forall X$, $ \forall d$, $\forall C, C'$ where $C \approx_d C'$, $f(C, X, d) = f(C', X, d)$. }

\section{Theorems and Proofs}

\subsection{About Calinski-Harabasz Index ($CH$) and Axioms}

\textbf{Theorem 1.} \textit{Calinski-Harabasz Index ($CH$) satisfies Scale Invariance.}

\textit{Proof. } Consider $C$ over $(X, d)$, and assume that $d'$ be a distance function satisfying $d' = \alpha d(x,y)$ for $\alpha > 0$. Then,
\[
CH(C, X, d') =  \frac{\sum_{i=1}^{|C|}|C_i|\alpha^2 d^2(c_i, c) / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} \alpha^2  d^2(x, c_i) / (|X| - |C|)} = \frac{ \alpha ^2}{\alpha^2 } CH(C, X, d) = CH(C, X, d).
\]
Therefore, $CH$ is scale-invariant. \hfill $\Box$

\textbf{Theorem 2.} \textit{$CH$ satisfies Richness.}

\textit{Proof. } Let $C$ be a clustering over $X$. Consider $d$ in which satisfies four conditions: (1) $d(c_i,c)=10$ where $c_i$ is a centroid of a cluster within $C$ (2) $d(c'_i,c)=1$ where $c'_i$ is a centroid of a cluster within any other possible clusterings except $C$ (3) $d(x, c_i)=1$ where $c_i$ is a centroid of a cluster within $C$ and $x$ is a member of that cluster (4) $d(x', c'_i)=10$ where $c'_i$ is a cluster within any other possible clusterings except $C$ and $x'$ is a member of that cluster. Then, $C = \arg\max_{C \in \mathcal{C}} f(C, X, d)$ where $\mathcal{C}$ is a set of all possible clusterings. \hfill $\Box$

\textbf{Theorem 3.} \textit{$CH$ satisfies Isomorphism Invariance. }

\textit{Proof. } $CH$ only considers the distances between data points, and does not regards their external identities. Therefore, for two isomorphic clustering ($C \approx_d C')$, $CH(C,X,d) = CH(C', X, d)$. \hfill $\Box$

\textbf{Theorem 4}. \textit{$CH$ does not satisfy Shift Invariance. }

\textit{Proof. }
Consider a clustering $C$ over $(X, d)$. Then, as
\[
CH(C, X, d + \beta) = \frac{\sum_{i=1}^{|C|}|C_i|(d(c_i, c) + \beta)^2 / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} (d(x, c_i)+\beta)^2 / (|X| - |C|)} \neq CH(C, X, d),
\]
$CH$ does not satisfy Shift Invariance.
\hfill $\Box$

\textbf{Theorem 5}. \textit{$CH$ does not satisfy Class-Cardinality Invariance.}

\textit{Proof.} $CH$ handles entire clusters within $C$ at once, rather than handling them in pairwise. Therefore, $CH$ does not satisfy Class-Cardinality Invariance.    \hfill $\Box$

\textbf{Theorem 6.} \textit{$CH$ does not satisfy Range Invariance.}

\textit{Proof. } The range of $CH$ is $(0, +\infty)$. Therefore, $CH$ is not range invariant.  \hfill $\Box$

\subsection{About Class-Cardinality Invariance}

Ackerman and Ben-David \cite{bendavid08nips} showed that if a function $f'$ satisfies within-dataset axioms, then its aggregation $f(C, X, d)  = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d)$ also satisfies the axioms.

\textbf{Theorem 7.} \textit{If $f'$ satisfies a within-dataset axiom, then $f(C, X, d)  = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d)$ also satisfies the axiom.} \cite{bendavid08nips}

Here, we provide a proof for between-dataset axioms.

\textbf{Theorem 8.} \textit{If $f'$ satisfies Data-Cardinality Invariance, then $f(C, X, d)  = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d)$ is also Data-Cardinality Invariance.}

\textit{Proof.} $f(C, X, d) = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d) = \text{agg}_{S \subseteq C, |S = 2|} f'(\underline{S}_\alpha, \underline{X}_\alpha, d) = f(\underline{C}_\alpha, \underline{X}_\alpha, d)$. Thus, $f$ satisfies Data-Cardinality Invariance. \hfill $\Box$

\textbf{Theorem 9.} \textit{If $f'$ satisfies Shift Invariance, then $f(C, X, d)  = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d)$ is also Shift Invariance.}

\textit{Proof.} $f(C, X, d) = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d) = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d + \beta) = f(C, X, d + \beta)$. Thus, $f$ satisfies Shift Invariance.

\textbf{Theorem 10.} \textit{If $f'$ satisfies Range Invariance, then $f(C, X, d)  = \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d)$ is also Range Invariance.}

\textit{Proof.} $\min_C f'(C, X, d) = 0$ implies $\min_C f(C, X, d) = \min_C \text{agg}_{S \subseteq C, |S = 2|} f'(S, X, d) = 0$. In the same way, $\max_C f'(C, X, d) = 1$ implies $\max_C f(C,X, d) = 1$. Thus, $f$ is Range Invariance.

\subsection{About Between-Dataset Calinski-Harabasz Index ($CH_{btwn}$) and Axioms}

\textbf{Theorem 11.} \textit{$CH_{btwn}$ satisfies Class-Cardinality Invariance.}

\textit{Proof.} By definition, $CH_{btwn}$ is an aggregation of $CH_3$. \hfill $\Box$

\textbf{Theorem 12.} \textit{$CH_{btwn}$ satisfies Scale Invariance.}

\textit{Proof. } Consider $C$ over $(X, d)$.
\begin{multline*}
    CH_1(C, X, \alpha d ) =
\frac{\sum_{i=1}^{|C|}|C_i|e^{(\alpha \cdot d(c_i, c)) / \sigma_{\alpha d}} / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} e^{(\alpha \cdot d(x, c_i))  / \sigma_{\alpha d }} / (|X| - |C|)} \\
=
\frac{\sum_{i=1}^{|C|}|C_i|e^{(\alpha \cdot d(c_i, c)) / (\alpha \cdot \sigma_{d})} / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} e^{(\alpha \cdot d(x, c_i))  / (\alpha \cdot \sigma_{d })} / (|X| - |C|)} = CH_1(C, X, d).
\end{multline*}
Therefore, $CH_1$ is scale invariant, and this makes $CH_2$ and $CH_3$, which are the functions that gets $CH_1$ as input, Scale Invariant. Then, as $CH_{btwn}$ is an aggregation of $CH_3$, due to Theorem 7, $CH_{btwn}$ is Scale Invariant. \hfill $\Box$

\textbf{Theorem 13. } \textit{$CH_{btwn}$ satisfies Richness}.

\textit{Proof.} Consider the same $C, X$, and $d$ with the proof of Theorem 2. In the case, $C = \arg\max_{C \in \mathcal{C}} CH_{btwn}(C, X, d)$ where $\mathcal{C}$ is a set of all possible clusterings. \hfill $\Box$

\textbf{Theorem 14.} \textit{$CH_{btwn}$ satisfies Isomorphism Invariance.}

\textit{Proof.} As $CH$ is, $CH_{btwn} $ only considers the distances between data points, and does not regards their external identities. Therefore, for two isomorphic clustering ($C \approx_d C')$, $CH_{btwn}(C,X,d) = CH_{btwn}(C', X, d)$. \hfill $\Box$

\textbf{Theorem 15.} \textit{$CH_{btwn}$ satisfies Shift Invariance. }

\textit{Proof.}
\begin{multline*}
    CH_1(C, X, d + \beta) =
\frac{\sum_{i=1}^{|C|}|C_i|e^{(d(c_i, c) + \beta) / \sigma_{d + \beta}} / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} e^{(d(x, c_i) + \beta) / \sigma_{d + \beta}} / (|X| - |C|)} \\
=
\frac{\sum_{i=1}^{|C|}|C_i|e^{d(c_i, c) / \sigma_{d}} / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} e^{d(x, c_i)/ \sigma_{d}} / (|X| - |C|)} \cdot \frac{e^\beta}{e^\beta} = CH_1(C, X, d).
\end{multline*}
Therefore, $CH_1$ is Shift Invariant.
This makes $CH_2$ and $CH_3$, which are functions that gets $CH_1$ as input, Shift Invariant. Then, as $CH_{btwn}$ is an aggregation of $CH_3$, due to Theorem 9, $CH_{btwn}$ is Shift Invariance. \hfill $\Box$

\section{Basis datasets Used in Validity Analysis}

\label{app:basis}

Following figure depicts a subset of basis dataset consists of two 2D Gaussian distribution used in validity analysis. We diversified the dataset in perspective of dimensionality, scale, number of points, and randomness to verify whether our between-dataset IVM satisfies the axioms.

\begin{figure}[h]
    \centering
    \includegraphics[width=\textwidth]{figures/basis.pdf}
    \caption{\mac{Do not use yellow with white background. Use blue/red instead} Subset of 1,000 2D basis datasets used in validity analysis, where each consists of two Gaussian distributions (left) and the dimension-augmented version of the datasets (right). For each 2D dataset, we randomized two distributions' size, variance, and a distance between them. To augment the dimensionality without disturbing the original cluster structure, we added Gaussian noise while maintain the variance of the noise smaller than the original distributions' variance. In this figure, only 3D augmented data was expressed, but data with a higher dimension was also created. }
    \label{fig:validity_data}
\end{figure}
\section{Detailed Settings in Between-dataset Rank Correlation Analysis}

\label{app:detail_btw}

\textbf{Classifiers settings. } As aforementioned, for consistency, we exploited the setting of Rodr\'iguez et al. \cite{rodriques18kbs}. The hyperparameters are set as follows:
\begin{itemize}
    \item \textbf{Support Vector Machine} (SVM): We used C-support vector classification model with default hyperparameter setting provided by \texttt{sklearn} \cite{pedregosa11jmlr}.
    \item \textbf{$k$-Nearest Neighbor} ($k$NN): We used $k=1$, and for all other parameters, we used default setting of \texttt{sklearn}. \mac{That is not really good use of KNN, several values should be tried}
    \item \textbf{Multilayer Perceptron} (MLP): We used 500 epochs, learning rate of 0.3, and momentum of 0.2. We used single hidden layer, where the size of the hidden layer is determined as the half the sum of dimension and number of classes. All other parameters are set to the default provided by \texttt{sklearn}.
    \item \textbf{Naive Bayesian networks} (NB): We used default parameter setting provided by \texttt{sklearn}.
    \item \textbf{Random forest} (RF): We used RF classifier consists of 100 trees with max depth of 10. Remaining parameters are set to the default of \texttt{sklearn}.
    \item \textbf{Logistic regression} (LR): We used 100 iterations. All remaining parameters are set to the default of \texttt{sklearn}.
    \item \textbf{Linear discriminant analysis} (LDA): We used least squares solution solver, and all other parameters are set to the default provided by \texttt{sklearn}.
\end{itemize}

We conducted 5-fold cross validation for each classifier, and used a mean value as a final classification score for a dataset. For the ensemble of classifiers, we used the max combination of the classifiers (i.e., chose the highest score among the classifiers).\mac{we miss the end of the story. we rank the scatterplot based on this classification score.}

\mac{For all supervised techniques, there should be a  grid search over the meta-parameters. you shall use Auto-ML or similar package for Python (pick the one with the most technique and the most recent): https://automl.github.io/auto-sklearn/master/
https://hyperopt.github.io/hyperopt-sklearn/
https://auto.gluon.ai/stable/tutorials/tabular\_prediction/tabular-quickstart.html
}

\textbf{Settings for computing ground-truth data-CLM.}
We used \texttt{sklearn} implementation for DBSCAN, $K$-Means, Birch, Agglomerative clustering, \texttt{sklearn-extra} implementation for $K$-Medoids, and \texttt{pyclustering} implementation for $X$-Means. For HDBSCAN, we used the implementation of McInnes et al. \cite{mcinnes17oss}.
To find the best hyperparameter setting for each clustering technique, we used Bayesian optimization \cite{snoek12neurips} implementation provided by Nogueira \cite{nogueira14github}. For each clustering techniques, we performed five random exploration and 25 steps of optimization, which are the defaults of the implementation.
The bounds of each technique's hyperparameter is set as follows:
\begin{itemize}
    \item \textbf{DBSCAN}
    \begin{itemize}
        \item \texttt{epsilon}: [0.01, 1.0]
        \item \texttt{min\_samples}: [1, 10]
    \end{itemize}
    \item \textbf{HDBSCAN}
    \begin{itemize}
        \item \texttt{epsilon}: [0.01, 1.0]
        \item \texttt{min\_samples}: [1, 10]
        \item \texttt{min\_cluster\_size}: [2, 50]
    \end{itemize}
    \item \textbf{\textit{K}-Means}
    \begin{itemize}
        \item $K$ (numver of clusters): [2, (number of classes of data) $\cdot$ 3]
    \end{itemize}
    \item \textbf{\textit{K}-Medoids}
    \begin{itemize}
        \item $K$: [2, (number of classes of data) $\cdot$ 3]
    \end{itemize}
    \item \textbf{\textit{X}-Means}
    \begin{itemize}
        \item \texttt{kmax}: [2, 50]
        \item \texttt{tolerance}: [0.01, 1.0]
    \end{itemize}
    \item \textbf{Birch}
    \begin{itemize}
        \item \texttt{threshold}: [0.1, 1.0]
        \item \texttt{branching\_factor}: [10, 100]
    \end{itemize}
    \item \textbf{Agglomerative Clustering}
    \begin{itemize}
        \item number of clusters: [2, (number of classes of data) $\cdot$ 3]
    \end{itemize}
\end{itemize}
After finding the best hyperparameter setting of a clustering technique for a given datasets, we ran the technique 20 times with same setting and report the average value as the final score.

\newpage
\section{Datasets used in Rank Correlation Analyses}

\label{app:datasets}

\mac{Add another table (two columns) showing all datasets ranked by their decreasing value of CHbtwn, together with the CHbtwn score}

\begin{table}[H]

  \caption{Real-World Datasets used in Between-dataset and Within-Dataset Rank Correlation Analyses }
  \label{dataset-table}
  \centering \small
  \begin{tabular}{lllll}
    \toprule
    Dataset & Objects & Features & Classes & Reference \\
    \midrule
        Birds Bones and Living Habits & 413 & 10 & 6 & Kaggle~\cite{kaggle} \\
        Cardiovascular Study Dataset & 2,927 & 15 & 2 & Kaggle \\
        Credit Risk Classification Dataset & 976 & 11 & 2 & Kaggle \\
        Customer Classification & 1,000 & 11 & 4 & Kaggle \\
        Fraud detection bank dataset 20K records binary & 20,468 & 112 & 2 & Kaggle \\
        Heart Attack Analysis \& Prediction Dataset & 303 & 13 & 2 & Kaggle \\
        League of Legends Diamond Games & 48,651 & 17 & 2 & Kaggle \\
        Microbes Dataset & 30,527 & 24 & 10 & Kaggle \\
        Mobile Price Classification & 2,000 & 20 & 4 & Kaggle \\
        Music Genre Classification & 1,000 & 26 & 10 & Kaggle \\
        Orbit Classification For Prediction / NASA & 1,748 & 11 & 6 & Kaggle \\
        Paris Housing Classification & 10,000 & 17 & 2 & Kaggle \\
        pH-recognition & 653 & 3 & 15 & Kaggle \\
        Rice Seed Dataset (Gonen\&Jasmine) & 18,185 & 10 & 2 & Kaggle \\
        Siberian Weather Stats & 1,439 & 11 & 9 & Kaggle \\
        Smoker Condition & 1,012 & 7 & 2 & Kaggle \\
        Water Quality & 2,011 & 9 & 2 & Kaggle \\
        Wine Customer Segmentation & 178 & 13 & 3 & Kaggle \\

        banknote authentication & 1,372 & 4 & 2 & UCI ML Repo~\cite{asuncion07uci} \\
        Breast Cancer Wisconsin (Prognostic) & 569 & 30 & 2 & UCI ML Repo \\
        Breast Tissue & 106 & 9 & 6 & UCI ML Repo \\
        CNAE-9 & 1,080 & 856 & 9 & UCI ML Repo \\
        Dermatology & 358 & 34 & 6 & UCI ML Repo \\
        Echocardiogram & 61 & 10 & 2 & UCI ML Repo \\
        Ecoli & 336 & 7 & 8 & UCI ML Repo \\
        Glass Identification & 214 & 9 & 6 & UCI ML Repo \\
        Harberman's Survival & 306 & 3 & 2 & UCI ML Repo \\
        Hepatitis & 80 & 19 & 2 & UCI ML Repo \\
        Image Segmentation & 210 & 19 & 7 & UCI ML Repo \\
        Ionosphere & 351 & 34 & 2 & UCI ML Repo \\
        Iris & 150 & 4 & 3 & UCI ML Repo \\
        Letter Recognition & 20,000 & 16 & 26 & UCI ML Repo \\
        MAGIC Gamma Telescope & 19,020 & 10 & 2 & UCI ML Repo \\
        Optical Recognition of Handwritten Digits & 3,823 & 64 & 10 & UCI ML Repo \\
        Pen-Based Recognition of Handwritten Digits & 7,494 & 16 & 10 & UCI ML Repo \\
        Planning Relax & 182 & 12 & 2 & UCI ML Repo \\
        SECOM & 1,567 & 590 & 2 & UCI ML Repo \\
        Spambase & 4,601 & 57 & 2 & UCI ML Repo \\
        SPECTF Heart & 80 & 44 & 2 & UCI ML Repo \\
        Statlog (German Credit Data) & 1,000 & 24 & 2 & UCI ML Repo \\
        Statlog (Image Segmentation) & 2,310 & 19 & 7 & UCI ML Repo \\
        Taiwanese Bankruptcy Prediction Data Set & 6,819 & 95 & 2 & UCI ML Repo \\
        Wine & 178 & 13 & 3 & UCI ML Repo \\
        Yeast & 1,484 & 8 & 10 & UCI ML Repo \\
        Zoo & 101 & 16 & 7 & UCI ML Repo \\

        Blood Transfusion Service Center & 748 & 4 & 2 & Yeh et al.~\cite{yeh2009knowledge} \\
        Boston & 154 & 13 & 3 & Sedlmair et al.~\cite{sedlmair2012taxonomy} \\
        Breast Cancer Coimbra & 116 & 9 & 2 & Patrício et al.~\cite{patricio2018using} \\
        Breast Cancer Wisconsin (Original) & 683 & 9 & 2 & Wolberg et al.~\cite{wolberg1990multisurface} \\
        CIFAR10 & 3,250 & 1,024 & 10 & Krizhevsky et al.~\cite{krizhevsky2009learning} \\
        Classification in asteroseismology & 1,001 & 3 & 2 & Hon et al.~\cite{hon2017deep} \\
        COIL20 & 1,440 & 400 & 20 & Nene et al.~\cite{nene20columbia} \\
        Crowdsourced Mapping & 10,545 & 28 & 6 & Johnson et al.~\cite{johnson2016integrating} \\
        Date Fruit Datasets & 898 & 34 & 7 & Koklu et al.~\cite{koklu2021classification} \\
        Diabetic Retinopathy Debrecen Data Set & 1,151 & 19 & 2 & Antal et al.~\cite{antal2014ensemble} \\
        Dry Bean Dataset & 13,611 & 16 & 7 & Koklu et al.~\cite{koklu2020multiclass} \\
        Durum Wheat Features & 9,000 & 236 & 3 & Kaya et al.~\cite{kaya2019towards} \\
    \bottomrule

  \end{tabular}

\end{table}

\begin{table}[H]
  \label{dataset-table2}
  \centering \small
  \begin{tabular}{lllll}
    \toprule
    Dataset & Objects & Features & Classes & Reference \\
    \midrule
        Epileptic Seizure Recognition & 5,750 & 178 & 5 & Andrzejak et al.~\cite{andrzejak2001indications} \\
        ExtyaleB & 319 & 30 & 5 & Georghiades et al.~\cite{georghiades2001few} \\
        Fashion-MNIST & 3,000 & 784 & 10 & Xiao et al.~\cite{xiao2017fashion} \\
        Fetal Health Classification & 2,126 & 21 & 3 & Ayres-de-Campos et al.~\cite{ayres2000sisporto} \\
        Flickr Material Database & 997 & 1,536 & 10 & Sharan et al.~\cite{sharan2009material} \\
        HAR & 735 & 561 & 6 & Anguita et al.~\cite{anguita2012human} \\
        Hate Speech & 3,221 & 100 & 3 & Davidson et al.~\cite{davidson2017automated} \\
        Heart Disease & 297 & 13 & 5 & Detrano et al.~\cite{detrano1989international} \\
        HIVA & 3,076 & 1,617 & 2 & Guyon et al.~\cite{guyon2007agnostic} \\
        HTRU2 & 17,898 & 8 & 2 & Lyon et al.~\cite{lyon2016fifty} \\
        Human Stress Detection & 2,001 & 3 & 3 & Rachakonda et al.~\cite{rachakonda2019stress} \\
        IMDB & 3,250 & 700 & 2 & Maas et al.~\cite{maas2011learning} \\
        Insurance Company Benchmark  & 5,822 & 85 & 2 & Putten et al.~\cite{van2004bias} \\
        Labeled Faces in the Wild & 2,200 & 5,828 & 2 & Huang et al.~\cite{huang2008labeled} \\
        Mammographic Mass & 830 & 5 & 2 & Elter et al.~\cite{elter2007prediction} \\
        Mnist64 & 1,082 & 64 & 6 &  Xia et al.~\cite{xia22tvcg} \\
        Olivetti faces dataset & 400 & 4,096 & 40 & AT\&T Lab. Cambridge~\cite{atntlab} \\
        Parkinsons & 195 & 22 & 2 & Little et al.~\cite{little2007exploiting} \\
        Patient Treatment Classification & 4,412 & 10 & 2 & Mujiono Sadikin~\cite{sadikin2020} \\
        Pima Indians Diabetes Database & 768 & 8 & 2 & Smith et al.~\cite{smith1988using} \\
        Pistachio Dataset & 2,148 & 28 & 2 & Ozkan et al.~\cite{ozkan2021classification} \\
        Pumpkin Seeds Dataset & 2,500 & 12 & 2 & Koklu et al.~\cite{koklu2021use} \\
        Raisin Dataset & 900 & 7 & 2 & Cinar et al.~\cite{ccinar2020classification} \\
        Rice Dataset Cammeo and Osmancik & 3,810 & 7 & 2 & Cinar et al.~\cite{cinar2019classification} \\
        seeds & 210 & 7 & 3 & Charytanowicz et al.~\cite{charytanowicz2010complete} \\
        Seismic Bumps & 646 & 24 & 2 & Sikora et al.~\cite{sikora2010application} \\
        Sentiment Labeled Sentences & 2,748 & 200 & 2 & Kotzias et al.~\cite{kotzias2015group} \\
        SkillCraft1 Master Table Dataset & 3,338 & 18 & 7 & Thompson et al.~\cite{thompson2013video} \\
        SMS Spam Collection & 835 & 500 & 2 & Almeida et al.~\cite{almeida2011contributions} \\
        Street View House Numbers & 732 & 1,024 & 10 & Netzer et al.~\cite{netzer2011reading} \\
        student grade pass or fail & 395 & 29 & 2 & Cortez et al.~\cite{cortez2008using} \\
        Turkish Music Emotion Dataset & 400 & 50 & 4 & Er et al.~\cite{er2019music} \\
        User Knowledge Modeling & 258 & 5 & 4 & Kahraman et al.~\cite{kahraman2013development} \\
        Weather & 365 & 192 & 7 & Ventocilla et al.~\cite{ventocilla2020comparative} \\
        Website Phishing & 1,353 & 9 & 3 & Abdelhamid et al.~\cite{abdelhamid2014phishing} \\
        Wilt & 4,339 & 5 & 2 & Johnson et al.~\cite{johnson2013hybrid} \\
        Wine Quality & 4,898 & 11 & 7 & Cortez et al.~\cite{cortez2009modeling} \\
        Wireless Indoor Localization & 2,000 & 7 & 4 & Rohra et al.~\cite{rohra2017user} \\
        World12d & 150 & 12 & 5 & Sedlmair et al.~\cite{sedlmair2012taxonomy} \\
    \bottomrule

  \end{tabular}

\end{table}
\section{Running Time Comparison}

\label{app:time}
\section{Detailed Explanation of Internal Measures}
\label{app:im}

\subsection{Internal Clustering Validation}
\label{sec:icv}

As clustering algorithm's goal is to maximize distinction between clusters and similarity within a cluster, most internal measures are designed to measure clusterings based on two criteria: separability and compactness \cite{liu10idcm, tan05idm, zhao02cikm, rendon11ac}. While separability denotes the degree to which clusters are apart from other clusters, compactness represents closeness of the points within a cluster.
Most measures are defined as the \textit{Ratio} (e.g., Calinski-Harabasz \cite{calinski74cis}, Xie-Beni \cite{xie91tpami}), \textit{Difference }(e.g., Silhouette \cite{rousseeuw87jcam}), or \textit{Sum }(e.g., $S\_Dbw$ \cite{halkidi01icdm}, $CVNN$ \cite{liu13tc}) of two criteria. Here, we list nine widely used internal measures surveyed by Liu et al. \cite{liu10idcm} and Xiong and Li. \cite{xiong13cvm}, and and organize them in three categories. In \autoref{sec:clustgen}, we demonstrate how these measures are generalized to clusterness measures. Note that we only considered monotonic measures, where optimal clustering is determined as the one with maximum or minimum score, instead of the shift point (i.e., the elbow) \cite{halkidi01jiis} which requires subjective determination to be judged. Exact definitions of the measures can be found in Table XXX.

\begin{table}
  \caption{Definition of commonly used internal measures (\autoref{sec:icv}) and their characteristics. Note that there is no fixed range for $SD$ and $CVNN$, as their range depends on normalization factors, which are determined based on currently compared clusterings.}
  \label{sample-table}
  \centering \small
  \begin{tabular}{cclcc}
    \toprule
    Type & Symbol & Definition & Optimal & Range \\
    \midrule
     Ratio & $CH$  & $\frac{\sum_i|C_i|d^2(c_i, c) / (NC-1)}{\sum_i \sum_{x\in C_i} d^2(x, c_i) / (|D| - NC)}$  & max & $(0, \infty)$ \\
     Ratio  & $I$  & $(\frac{1}{NC} \cdot \frac{\sum_{x \in D} d(x, c)}{\sum_i \sum_{x \in C_i} d(x, c_i)} \cdot \max_{i, j} d(c_i, c_j)) ^ p$ ($p=2$ for default)   & max & $(0, \infty)$ \\
     Ratio & $D$  &  $\frac{\min_{i,j} \min_{x \in C_i, y \in C_j} d(x, y)}{\max_k \max_{x, y \in C_k} d(x, y)}$ & max & $(0, \infty)$  \\
    Ratio & $XB$ &  $ \frac{\sum_i \sum_{x \in C_i} d^2(x, c_i)}{|D| \cdot \min_{i, j \neq i} d^2(c_i, c_j)}$ & min & $(0,\infty)$ \\
     Ratio & $DB$   & $\frac{1}{NC} \cdot \sum_i \max_{j\neq i}\frac{\sum_{k =\{i, j\}}(\frac{1}{|C_k|}\sum_{x \in C_k} d(x, c_k))}{d(c_i, c_j)} $ & min & $(0, \infty)$  \\
    Sum & $S\_Dbw$  & $Scat + Dens\_bw$ & min & (0, 2)\\
        &           & $Scat = \frac{1}{NC} \sum_i \| \sigma(C_i) \| / \| \sigma(D) \|$ & &  \\
        &           & $Dens\_bw = \frac{1}{NC \cdot (NC-1)} (\sum_{i, j \neq i} \frac{density(u_{ij})}{\max(density(c_i), density(c_j))})$ &  & \\
    Sum  & $SD$ & $\alpha \cdot Scat + Dis$ & min & - \\
       &        & $Dis = \frac{\max_{i, j} d(c_i, c_j)}{\min_{i, j} d(c_i, c_j)} \cdot \sum_i (\sum_j d(c_i, c_j))^{-1}$ & & \\
    Sum & $CVNN$   & $Sep(k) / \max_{NC} + Com / \max_{NC}$ & min & - \\
    & & $Sep = \max_i(\frac{1}{|C_i|} \sum_{j=1, 2, \cdots, |C_i|} \frac{q_j}{k}) $ & & \\
    & & $Com = \sum_i(\frac{2}{|C_i| \cdot (|C_i| -1 ) } \sum_{x,y \in C_i} d(x,y))$ & & \\
    Diff  & $SC$ & $\frac{1}{NC} \cdot \sum_i [\frac{1}{|C_i|} \cdot\sum_{x \in C_i} \frac{b(x) - a(x)}{\max(b(x), a(x))}]$ & min & (-1, 1) \\
       &  & $a(x) = \frac{1}{|C_i| - 1} \sum_{y \in C_i, y \neq x} d(x, y)$ & & \\
       &  & $b(x) = \min_{j \neq i} (\frac{1}{|C_j|} \sum_{y \in C_j} d(x,y))$ & &\\
    \bottomrule
    \addlinespace[0.115cm]
    \multicolumn{5}{l}{
        \footnotesize
        $D$: dataset; $C_i$: $i$-th cluster; $c / c_i$: centroid of dataset / $i$-th cluster; $NC$: number of clusters;
    } \\
    \multicolumn{5}{l}{
        \footnotesize
         $d(x, y)$: distances between points $x$ and $y$; $v(C)$: varaince vector of $C$; $\| v \| = \sqrt{v^T \cdot v}$;
    } \\
    \multicolumn{5}{l}{
        \footnotesize
         $u_{ij}$: midpoint of $c_i$ and $c_j$; $density(u_{ij}) = \sum_{x \in D} f(x, u_{ij})$, $density(c_i) = \sum_{x \in C_i} f(x, c_i)$;
    } \\
    \multicolumn{5}{l}{
        \footnotesize
         $f(x, c) = 0$ if $d(x, c) > \frac{1}{NC} \sqrt{\sum_i\| \sigma(C_i) \|}$ and else 1; $k$: number of considered nearest neighbors
    } \\
    \multicolumn{5}{l}{
        \footnotesize
         $q_j$: number of $C_i$'s $j$-th point's nearest neighbors that are not belong to $C_i$
    } \\
  \end{tabular}

\end{table}

\textit{Ratio. }
Calinski-Harabasz index ($CH$) \cite{calinski74cis} evaluates compactness as a sum of squared distances each each point to its cluster centroid, and separability as a sum of squared distances of each cluster centroids to the global centroid of entire dataset.
I Index ($I$) \cite{maulik02tpami} computes compactness as a ratio of sum of distances of points to its cluster centroid to the sum of points to the global centroid. Separability is defined as a sum of pairwise distances between centroids.
In Dunn's index ($D$) \cite{dunn74joc}, compactness is  maximum intra-cluster distance and separability is minimum inter-cluster distance.
Xie-Beni Index ($XB$) \cite{xie91tpami} uses same definition for compactness with $CH$, while defining separability as minimum pairwise distances between centroids.
For Davies-Bouldin Index ($DB$), the similarity of two clusters are first defined where compactness is a sum of distances of points to their cluster centroid, and separability is a distance between centroids. Then, final score is computed as an average of the score assigned to each cluster; each cluster's score corresponds to the max similarity with among the ones with other clusters.
These measures are defined in a form of $Score = (a \cdot Separability) / (b \cdot Compactness)$ ($CH$, $I$, $D$) or its inverse ($XB$, $DB$). While the higher value implies better clustering in the former, lower value implies better clustering in the latter.

\textit{Summation. }
S\_Dbw Index ($S\_Dbw$) \cite{halkidi01icdm} evaluates compactness as a ratio of the average of cluster variance to the entire variance of the dataset. Separability instead takes the concept of density; for each pair of clusters, separation is defined as a ratio of density around midpoint of the cluster centroid to the density around centroids. SD Index ($SD$) \cite{halkidi00pkdd}  shares the definition of compactness with $S\_Dbw$, while utilizes the sum of cluster centroids to represent separation. Clustering Validation index based on Nearest Neighbors ($CVNN$)  \cite{liu13tc} compute separability as an average ratio of each points' nearest neighbors belonging to the same cluster to the total number of nearest neighbors. Compactness is computed as an average intra-cluster pairwise distances.
These three measures simply compute the final score as a sum of separability and compactness. However, as the range of separability and compactness can vary, the measures often requires additional adjustment. For example, in $SD$, additional weighting factor is multiplied to compactness to balance it with separability. In contrast, $CVNN$ normalizes both separability and compactness to the maximum separability and compactness computed among all clusterings that are currently compared.

\textit{Difference. }
Silhouette coefficient ($SC$) \cite{rousseeuw87jcam} quantifies average difference between mean intra-cluster distance (compactness) and the mean nearest-cluster distance (separability) for every point. The optimal clustering is determined as the one having the max index.

As internal validation measures are naturally designed to find optimal clustering , most of them cannot be used across datasets.
However, at least $S$ have been used across dataset; it is widely used to evaluate DR by quantifying the extent to which a cluster structure of the original space are well represented in the embeddings \cite{joia11tvcg, fadel15neurocomp}. Still, this use case is about comparing datasets with identical dimension, number of points and clusters.
Our generalization instead allows the measure to be used across arbitrary datasets, thus enables various useful application scenarios.

\section{Introduction}

\end{document}

\section{Introduction}

\label{sec:intro}

Cluster analysis \cite{jain88prenticehall} is an essential exploratory task for data scientists and practitioners in various application domains \cite{lyi15bioinformatics, schaeffer07gc, caron18eccv}. It relies on clustering techniques, that is, unsupervised machine learning algorithms that partition the data into subsets called groups or clusters, while maximizing between-cluster separation and within-cluster compactness based on some distance function \cite{liu10idcm}.

Clustering validation measures \cite{farber10multiclust} or quality measures \cite{bendavid08nips} have been proposed to evaluate clustering results. They can be \textit{internal} or \textit{external} \cite{rendon11ac, liu10idcm, jain88prenticehall}.
Internal validation measures (IVM) give high scores to partitions in which data points with high or low similarities to each other are assigned to the same or different clusters, respectively. In contrast, External validation measures (EVM) quantify how well a clustering matches a ground truth partition. \maa{Taking the classes of labeled data as ground truth is a widely used approach to rank clustering techniques on benchmark datasets \cite{farber10multiclust}.

\autoref{fig:clm} illustrates the main issue we propose to address in this work. Using class labels as ground truth in EVM relies on the Cluster-Label Matching (CLM) assumption that the dataset has a good matching between clusters and class labels \cite{aupetit14beliv} (\autoref{fig:clm}A).
\maa{In the worst case, the CLM of the data can be bad with data ranging from having labels randomly assigned to or split between easy-to-detect clusters (\autoref{fig:clm}B), to having labels being well assigned to hard-to-detect clusters (\autoref{fig:clm}J). Then, a low EVM score can be due to a bad clustering of an otherwise good-CLM dataset (\autoref{fig:clm}G) (the clustering technique has low capacity to detect complex clusters) or to a bad-CLM dataset (\autoref{fig:clm}BJ) (the clustering technique may as well have a high capacity (\autoref{fig:clm}CE) or a low one (\autoref{fig:clm}FH), we cannot tell; EVM is unreliable when CLM is bad). Thus, it is crucial to evaluate CLM to measure the intrinsic quality of the ground truth dataset in order to inform and weigh the results of the EVM accordingly (\autoref{sec:selection}).} \rev{Still, the results of EVM over benchmark datasets are often given without considering their CLM \cite{rehioui16pcs, khan21icecit, monath17nips, monath19kdd} casting doubts on the rankings obtained}. \maa{Here, we aim to evaluate the CLM of labeled datasets}.

\begin{figure}[t]
    \centering
    \includegraphics[width=0.88\textwidth]{figures/clm.pdf}\\

    \caption{An External Validation Measure (\blue{EVM}) evaluates the matching between two partitions (markers' shape and fill) of the same data (D, E, G, H). The Cluster-Label Matching (\brown{CLM}) is \green{good} (A, C) if the partition formed by the \textit{labels} assigned to the data (shape or fill) matches well the \textit{clusters} formed by the data point distribution (encoded by position). The labels can be provided with the \textit{data} (fill) (A, B) or as the output partition of the \textit{clustering} technique to be evaluated (shape)(C, F). If \brown{\textit{data-CLM}} is \green{good} (A), the \blue{EVM} between the data labels (A) and clustering labels (C, F) gives a \green{\textit{reliable}} evaluation (D, G) of the \brown{\textit{clustering-CLM}}: \green{high}/\red{low} \blue{EVM} (D/G) match with \green{good}/\red{bad} \brown{clustering-CLM} (C/F). But if \brown{data-CLM} is \red{bad} (B), the EVM is always \red{low} (E, H) and \red{\textit{unreliable}} to evaluate if the \brown{clustering-CLM} is good (C) or bad (F). It is highly unlikely to get a high EVM (as in D) if the \brown{data-CLM} is bad (B) as it would mean the clustering technique found by chance the same \textit{bad partition} as the one given by the data labels (fill (B) and shape (F) would match perfectly in H, not shown).
    \rev{Our main goal is to evaluate and compare \brown{data-CLM} of \textit{several datasets} (I,J,K,L) with different characteristics (dimension, size, data and class distributions)
    }\maa{to inform clustering evaluation with EVMs. }\rev{ But how can we get CLM scores comparable between datasets?}}
    \label{fig:clm}

\end{figure}

Yet, evaluating and comparing the CLM of benchmark datasets is challenging. We can use the \ma{average EVM scores of several} clustering techniques or the accuracy of classifiers in distinguishing classes \cite{rodriques18kbs, abul03smc} as a proxy for CLM, but such approaches are not based on principled axioms and are also very time-consuming. \ma{In contrast, IVMs are easy to compute, can derive from axioms~\cite{bendavid08nips} and} \ma{could be used as a proxy for CLM, as class partitions forming good clusters would get a higher score.
However, they are designed to compare different clustering partitions of a \textit{single} dataset (\autoref{fig:clm} C \textit{vs} F), rather than the class partition of \textit{different} datasets (\autoref{fig:clm}IJKL). As datasets can differ by their size, dimension, data and class distributions, IVM scores of two labeled datasets \maa{cannot be reliably compared, making IVM improper measures of CLM across datasets; such claim is also confirmed by our experiments} (\autoref{sec:bdsa}; \autoref{tab:dataset_entire}).
Thus, we lack a proper measure to compare CLM across datasets.}

In this research, we \ma{propose a set of new axioms from which we derive} a \textit{between-dataset} internal validation measure (IVM$_{btwn}$) as a grounded way to assess and compare CLM of different datasets.
\rev{An IVM$_{btwn}$ takes a single labeled dataset as input and returns a score evaluating its level of CLM. This score is designed to be comparable across datasets.}
Our contribution is four-fold:

\textbf{Axioms } We propose four \emph{between-dataset axioms} that IVM$_{btwn}$  should satisfy for the fair comparison of CLM, complementing Ackerman and Ben-David's \emph{within-dataset} axioms \cite{bendavid08nips} satisfied by standard IVM (i.e., within-dataset IVM; IVM$_{wthn}$\xspace) (\autoref{sec:axiom}). These additional axioms require the IVM$_{btwn}$ to be invariant to the number of data points, classes, and dimensions, and to share a common range.

\textbf{Generalization process and new between-dataset Calinski Harabasz index} From these axioms, we propose technical tricks for generalizing an IVM$_{wthn}$\xspace into an IVM$_{btwn}$\xspace. We use them to generalize the Calinski-Harabasz index  ($CH$) \cite{calinski74cis} into  a \emph{between-dataset} $CH$ index  ($CH_{btwn}$) (\autoref{sec:ivmbtw}).

\textbf{Quantitative evaluations} Through an ablation study, we verify the validity and necessity of our generalization process (\autoref{sec:validity}). We also show that $CH_{btwn}$ \ma{ranking 96 real-world datasets significantly outperforms competitors in terms of rank-correlation with the ground truth CLM} \rev{approximated} \ma{based on nine clustering techniques, while being up to three orders of magnitude faster to compute than the} \rev{approximate} \ma{ground truth (Appendix D). These experiments demonstrate} \rev{the validity of our axiomatic approach and the effectiveness of $CH_{btwn}$ (\autoref{sec:bdsa}).}

\textbf{Ranking real benchmark data for reliable EVM} Lastly, we explain the importance of evaluating CLM in advance of external validation by showing how not \maa{doing so} adversely affects the conclusions about the comparative performances of clustering techniques (\autoref{sec:selection}).

\section{\rev{Backgrounds and} Related Works}

Many clustering techniques exist \rev{\cite{xu15ads}}, and ensemble approaches have been proposed to combine clustering results to compensate for the weaknesses of individual techniques \cite{vega11jprai}. Still, it is challenging to define what a \textit{good} clustering is.
For instance, stability is deemed an important criterion \cite{von10, ben06icclt}.

EVM quantify how much the resulting clustering matches with a ground truth partition of the data. For example, Adjusted Mutual Information  \cite{strehl2002jmlr, vinh10jmlr} measures the agreement of two label assignments in terms of information gain corrected for chance effects. Other measures, such as Adjusted Rand Index \cite{santos09icann} or V-measure \cite{rosenberg07emnlp}, can be used instead.

Classes of labeled datasets have been used extensively as ground truth for clustering EVM \cite{farber10multiclust}.
\maa{However, despite its potential risk of violating CLM, no principled procedure has yet been proposed to evaluate the reliability of such a ground truth. Our research aims to fill this gap by proposing a measure of CLM. A similar endeavor has been engaged in the supervised learning community to quantify datasets' difficulty for classification tasks \cite{pmlr-v162-ethayarajh22a}.}

A natural idea would be to use classification scores as a proxy for CLM \cite{abul03smc, rodriques18kbs}. This approach is based on the assumption that the classes of a labeled dataset getting good classification scores will provide a reliable ground-truth for EVM. Still, a classifier is not designed to distinguish well between \ma{two "adjacent" classes forming a single cluster (\autoref{fig:clm}B light blue and purple bottom left cluster, good class separation but bad CLM) and two "separated" classes forming distant clusters (\autoref{fig:clm}A light blue and orange clusters, good CLM), nor it is designed to distinguish within-class structures like a class forming a single cluster (\autoref{fig:clm}A light blue class, good CLM) and one made of several distant clusters (\autoref{fig:clm}B light blue class, bad CLM)}. Moreover, classifiers require expensive training time \ma{(Appendix F)}.

A more direct approach is to average the results of multiple and diverse clustering techniques \cite{vega11jprai} as their high EVM scores would indicate a good CLM (\autoref{fig:clm}D). However, this approach is computationally expensive too \ma{(Appendix F)}. Moreover, the ground truth it \rev{approximates} is not based on principled axioms independent of any clustering technique, so it is likely biased in regard to the certain type of clusters these techniques can detect. For lack of a better option, though, we use this approach to \rev{get an approximate} ground truth in our experiments validating our axiom-based solution, while mitigating the bias by aggregating the EVM scores of multiple clustering techniques.

In contrast to classifiers or clustering techniques, most IVM are relatively inexpensive to compute \ma{(Appendix F)}. Also, as they are based on two criteria---\textit{compactness} (i.e., the pairwise closeness of data points within a cluster) and \textit{separability} (i.e., the degree to which clusters lie apart from one another) \cite{liu10idcm, tan05idm, zhao02cikm, rendon11ac}---they can examine the cluster structure in more details; in \autoref{fig:clm}, an IVM would give a higher score to partitions A and C than to B and F.
Moreover, following the axiomatization of clustering by Kleinberg \cite{kleinberg02nips}, Ackerman and Ben-David \cite{bendavid08nips} proposed four within-dataset axioms that give a common ground to all IVMs: scale invariance, consistency, richness, and isomorphism invariance. These axioms set the requirements a function should satisfy to work properly as an IVM.

Nevertheless, IVMs were originally designed to compare and rank different partitions of the \emph{same} dataset as shown in \autoref{fig:clm}A-H. Therefore, IVM cannot be used to compare CLM \emph{across} different datasets in which not only the cluster structure but also the number of points, classes, and dimensions can vary (\autoref{fig:clm}I-L).
Here, we propose four additional axioms that an IVM should satisfy to allow this comparison, derive a new IVM satisfying them, and apply it to rank labeled datasets by their reliability to be used as a basis for clustering EVM.

\section{New Axioms for Internal Clustering Validation}

\label{sec:axiom}

\subsection{Ackerman and Ben-David's \emph{Within-dataset} Axioms}

Ackerman and Ben-David (\AB) proposed \textit{within-dataset} axioms \cite{bendavid08nips} that specify the requirements for IVM to properly evaluate clustering partitions. The first axiom is \textbf{W1: Scale Invariance}; it requires measures to be invariant to distance scaling. \textbf{W2: Consistency} is satisfied by a measure that increases when within-cluster compactness or between-cluster separability increases. \textbf{W3: Richness} requires measures to give any fixed cluster partition the best score over the domain by only modifying the distance function. Lastly, \textbf{W4: Isomorphism Invariance} ensures that an IVM does not depend on points identity.
Detailed definitions are given in Appendix A.

\subsection{Axioms for Enabling \emph{Between-dataset} Comparison}

Within-dataset axioms do not consider the case of comparing scores across datasets; they assume the dataset is invariant.
We propose four additional \emph{between-dataset} axioms that should be satisfied by internal validation measures to allow a fair comparison of cluster partitions across datasets.

\textbf{Notations }
\rev{We follow the notations used in \AB.}
We define a finite domain set $X \subset \mathcal{D}$ of dimension $\Delta_X$, where $\mathcal{D}$ denotes data space.
We denote a clustering partition of $X$ as $C =\{C_1, C_2, \cdots, C_{|C|}\}$, where  $\forall i\neq j, C_i \cap C_j = \emptyset$ and $\cup_{i=1}^{|C|} C_i = X$.
A distance function $d : \mathcal{D} \times \mathcal{D} \rightarrow \mathbb{R}$ is a function that satisfies $d(x, y) \geq 0$, $d(x, y) = d(y, x)$ and $d(x, y) = 0$ if $x=y$ for any $x, y \in \mathcal{D}$.
If two point sets $X$ and $Y$ follow the same distribution, we say $X \overset{D}{=} Y$.
A measure is a function $f$ that takes $C, X, d$ as input and returns a real number. Throughout the section, higher $f$ implies better clustering.
Additionally, we define $\underline{W}_\alpha$ a random subsample of the set $W$ ($\underline{W}_\alpha\overset{D}{=}W$) such that $|\underline{W}_\alpha|/|W|=\alpha$, and $\underline{C}_\alpha=\{\underline{C_i}_\alpha\}_{i\in 1\dots |C|}$.

\textbf{Goals and factors at play}
IVM$_{wthn}$ operate on fixed dataset $X$  with possible variations of $C$ and distance $d$. Hence, the number $|C|$ and sizes $|C_i|{\forall i}$ of the generated clusters can vary, while $X$ determines a common basis for comparison. \AB's \rev{within-dataset} axioms essentially state that the measure $f$ should be invariant to various aspects of the distance $d$. Hence, as $X$ is fixed, $f$ can only vary in relation to the clustering partition $C$. The satisfaction of the \AB's axioms is a way to ensure  IVM$_{wthn}$\xspace focus on measuring the clustering quality and nothing else.

In contrast, between-dataset IVM (IVM$_{btwn}$\xspace) shall operate on varying $C$, $d$, and $X$. Imposing IVM$_{btwn}$\xspace to satisfy \AB's \rev{within-dataset} axioms will reduce the influence of $d$. Still  several aspects of the varying datasets $X$ now come into play and their influence on IVM$_{btwn}$\xspace shall be minimized. The sample size $|X|$ is one of them (\textbf{Axiom B1}) and the dimension $\Delta_X$ of the data another one (\textbf{Axiom B2}).
Moreover, what matters is the \textit{matching} between natural clusters and data labels more than the number of clusters or labels; therefore,
we shall reduce the influence of the number of labels $|C|$ (\textbf{Axiom B3}) imposed by the dataset. Lastly, we need to align IVM$_{btwn}$\xspace to a comparable range of values (\textbf{Axiom B4}) across datasets, in essence capturing all remaining hard-to-control factors unrelated to clustering quality (i.e., the \textit{matching} between natural clusters and data labels (CLM)) but integrated by the measure.

\textbf{Axiom B1} Invariance to the sample size is ensured if subsampling all clusters in the same proportion does not affect the IVM$_{btwn}$\xspace score, leading to the first axiom:

\textbf{\textit{Data-cardinality Invariance}} \textit{A measure $f$ satisfies data-cardinality invariance if
$\forall X,\forall d$ and for every clustering $C$ of $(X,d)$, $f(C,X,d) = f(\underline{C}_\alpha,X_\alpha,d)$ with $X_\alpha=\cup_{i=1}^{|C|} \underline{C_i}_\alpha$.}

\textbf{Axiom B2} We shall take into account that data dimension $\Delta_X$ may vary across datasets. An important aspect of the dimension called the concentration of distance phenomenon, which is related to the curse of the dimensionality \cite{bellman66science}, affects the distance measures involved in IVM$_{btwn}$\xspace. As dimension grows, the variance of data pairwise distances for any distribution tends to be constant while their mean value increases \cite{beyer99icdt, francois07tkde, lee11iccs}. Therefore, in high dimensional spaces,  $d$ will act as a constant function for any data $X$, thus an IVM$_{btwn}$\xspace $f$ will generate similar scores for all datasets.
To mitigate this phenomenon, and as a way to reduce the influence of the dimension, we require the measures to be shift invariant \cite{lee11iccs, lee14cidm} so that the shift of the distances (i.e., growth of the mean) is canceled out:

\textbf{\textit{Shift Invariance }} A measure $f$ satisfies shift invariance if $\forall X, \forall d$ and for every clustering $C$ over $(X, d)$, $f(C, X, d) = f(C, X, d+ \beta)$ $ \forall \beta > 0$ where $d + \beta$ is a distance function satisfying $(d + \beta)(x,y) = d(x,y) + \beta$, $\forall x, y \in X$.

\textbf{Axiom B3} The number of classes should not affect an IVM$_{btwn}$\xspace, for example, two well clustered classes should get an IVM$_{btwn}$\xspace score as good as 10 well clustered classes. \AB proposed to aggregate class-pairwise IVM$_{wthn}$\xspace  to form other valid IVM$_{wthn}$\xspace. We follow this principle but state it as an axiom for IVM$_{btwn}$\xspace:

\textbf{\textit{Class-cardinality Invariance} }
\textit{A measure $f$ satisfies class-cardinality invariance if $\forall X,  \forall d$ and $\forall C$ over $(X, d)$, $f(C, X, d) = $ }$ \textrm{agg}_{S \subseteq C, |S| = 2} f'(S, X, d)$ \textit{with}  $\textrm{agg}_S\in\{\textrm{avg}_S,\min_S,\max_S\}$ \textit{and $f'$ is an IVM.
}

By design, $f$ will satisfy all within or between axioms that $f'$ satisfies (Appendix B).

\textbf{Axiom B4} Lastly, we need to ensure that IVM$_{btwn}$\xspace  take a common range of values across datasets, so that their minimum and maximum values correspond to datasets with the worst and the best CLM, respectively, and that these extrema are aligned across datasets \ma{(we set them arbitrarily to 0 and 1)}:

\textbf{\textit{Range Invariance.}} \textit{A measure $f$ satisfies range invariance if $\forall X, \forall d$, and $C$ any clustering over $(X,d)$, $\min_{C} f(C,X,d)=0$ and $\max_{C} f(C,X,d)=1$.
}

\section{Generating Between-dataset Internal Validation Measures}

\label{sec:ivmbtw}

We propose technical tricks to generate IVM$_{btwn}$\xspace{}   \ma{that satisfy our supplementary axioms, and use these tricks} to generalize the Calinski-Harabasz index ($CH$) \cite{calinski74cis} to the between-dataset $CH$ index ($CH_{btwn}$).

\subsection{Generalization Tricks \rev{for Enabling Between-dataset Comparison}}

\textbf{Trick T1: Approaching data-cardinality invariance (B1) } We cannot guarantee the invariance of a measure for any subsampling of the data (e.g., very small sample size), but we can get some robustness to random subsampling if we use consistent estimators of population statistics  as building blocks of the measure, such as the mean or the median of a class, a pair of classes, or of the whole dataset, or quantities derived from them such as the average distance between all points of two classes.

\textbf{Trick T2: Achieving shift invariance (B2) }
Considering a vector of distances $u=(u_1\dots u_n)$, we can define a shift invariant measure by using a ratio of exponential functions $g_j(u)=\frac{e^{u_j}}{\sum_k e^{u_k}}$. We observe that $\forall S\in \mathbb{R}, g_j(u+S) = \frac{e^{u_j+S}}{\sum_k e^{u_k+S}}=\frac{e^{u_j}}{\sum_k e^{u_k}}\frac{e^{S}}{e^{S}}=g_j(u)$, hence $g_j$ is shift invariant. This trick is at the core of the $t$-SNE loss function \cite{van08jmlr,lee11iccs}.
However, $g_j$ is not scale-invariant: $\forall \lambda\in \mathbb{R},g_j(\lambda u) = \frac{e^{\lambda u_j}}{\sum_k e^{\lambda u_k}}\neq\lambda g_j(u)$, hence it will not satisfy axiom W1. We can get back scale-invariance by normalizing each distance $u_i$ by a term that scales with all of them together, for example, their standard deviation: $\sigma(u)$. Now $g_j(\lambda u/\sigma(\lambda u))=g_j(\lambda u/\lambda \sigma(u))=g_j(u/\sigma(u))$ is both shift and scale invariant.

\textbf{Trick T3: Achieving class-cardinality invariance (B3) }
Class-cardinality invariance can be achieved by following the definition of Axiom B3, such as by defining the measure $f$ as the average of class-pairwise measures.

\textbf{Trick T4: Achieving range invariance (B4) }
A common approach to get a unit range for $f$ is to use min-max scaling $f_u=(f-f_{\min})/(f_{\max}-f_{\min})$.
However, determining the possible minimum and maximum values of $f$ for any data $X$ is not straightforward. Theoretical extrema are usually computed for edge cases far from realistic $X$ and $C$. Wu et al. \cite{wu09kdd}  propose to estimate the worst score over a given dataset $X$ by the expectation $\hat{f}_{\min}=E_{\pi}(f(C^{\pi},X,d))$ of $f$ computed over random partitions $C^{\pi}$ of $(X,d)$ preserving class proportions $|C^{\pi}_i|=|C_i|{\forall i}$ (Trick 4a)---arguably the worst possible clustering partitions of $X$. In contrast, it is hard to estimate the maximum achievable score over $X$, as this is the very objective of clustering techniques. If the theoretical maximum $f_{\max}$ is known and finite, we propose to use it by default; otherwise, if infinite, we propose to use a logistic function (Trick 4b) to scale it down to $1$ (Note that the scaled measure $f_u$ is $0$ if $f_{\max}\rightarrow+\infty$).

\subsection{Generalizing the Calinski-Harabasz Index}

\label{sec:gener}

As a proof-of-concept, we use the proposed tricks to generalize the $CH$ index to the $CH_{btwn}$ index that satisfies \ma{both within-dataset (W) and between-dataset (B)} axioms. We select $CH$ as it is a representative  IVM$_{wthn}$\xspace \cite{liu10idcm, liu13tc,  maulik02tpami, xiong13cvm} widely used for clustering evaluation \cite{wang19mse, lukasik15cec, baarsch12imecs}. It is defined as:
\[
 CH(C, X, d) = \displaystyle\frac{\sum_{i=1}^{|C|}|C_i|d^2(c_i, c) / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} d^2(x, c_i) / (|X| - |C|)},
\]
where $c_i$ is the centroid of $C_i$ and $c$ is the \rev{centroid} of $X$. A higher value implies a better CLM. The denominator and numerator measure compactness and separability, respectively. Both are estimators of population statistics (Trick 1), reducing by design the influence of data-cardinality (Axiom B1).
We get shift invariance (Axiom B2) while preserving scale invariance (Axiom W1) by substituting the square distances by their exponential form normalized by the standard deviation $\sigma_{d}$ of the distances of data points to the centroid (Trick 2), leading to:
\[
 CH_1(C, X, d) = \frac{\sum_{i=1}^{|C|}|C_i|e^{d(c_i, c) / \sigma_{d}} / (|C| - 1)}{\sum_{i=1}^{|C|} \sum_{x \in C_i} e^{d(x, c_i) / \sigma_{d}} / (|X| - |C|)}.
\]

Then, we apply min-max scaling (Axiom B4).
As $\max(CH_1) = +\infty$, we transform it through a logistic function (Trick 4b) $CH_2 = 1 / (1 + CH_1^{-1})$ so $CH_{2\max}=1$. We estimate the worst score as the expectation of $CH_2$ over random clustering partitions $C^\pi$ (Trick 4a): $CH_{2\min}=E_\pi(CH_2(C^{\pi}, X, d))$. We get $CH_3= (CH_2 - CH_{2\min}) / (CH_{2\max} - CH_{2\min})$.

Lastly, we satisfy class-cardinality (Axiom B3) by averaging class-pairwise scores (Trick 3), which determines the between-cluster Calinski-Harabasz index:
\[
CH_{btwn}(C, X, d) = \frac{1}{{|C| \choose 2}}\sum_{S \subseteq C, |S| = 2} CH_3(S, X, d).
\]

Unlike $CH$, which misses all between-dataset axioms except B1, $CH_{btwn}$ satisfies all of them by design, and we prove it also satisfies all within-dataset axioms (Appendix B).

The existence of at least one IVM$_{btwn}$\xspace provides evidence pointing toward the consistency of our axioms. Still, we cannot prove their completeness nor their soundness for lack of a clear definition of what a good clustering is (See \AB \cite{bendavid08nips} for a discussion of these concepts for clustering).
Our experiments validate the importance of these axioms for comparing the CLM of different datasets.

In terms of computational complexity,
\rev{$CH$, $CH_1$, and} $CH_2$ are $O(|X|\Delta_X)$, thus $CH_{2\min}=O(|X|\Delta_X T)$ while $CH_{2\max}=O(1)$, where $T$ is the number of Monte Carlo simulations to estimate $CH_{2\min}$. Hence, $CH_3=O(|X|\Delta_X T)$, and finally  $CH_{btwn}=O(|X|\Delta_X T P_C)$, where $P_C=|C|(|C|+1)/2$ is the number of pairs of classes.
Worst-case complexity of $CH_{btwn}$ is linear with all parameters but quadratic with the number of classes, making it very scalable (Appendix F).

\section{Evaluation}

\subsection{Ablation Study of Between-dataset Calinski-Harabasz index}
\label{sec:validity}

\begin{figure}[t]
    \centering
    \includegraphics[width=\textwidth]{figures/validity_update.pdf}
    \vspace{-4mm}
    \caption{
    Ablation study of $CH_{btwn}$\xspace (\autoref{sec:validity}). Heatmaps (a--h) show the SMAPE of $CH_v$ variants (each column) for all pairs $(N_a,N_b)$ of controlled dataset sizes (top row) and pairs $(\Delta_a,\Delta_b)$ of controlled dimensions (bottom row). The lighter the color, the lower the error and the less sensitive $CH_v$ to variations of the controlled factor. Bar charts (right) show the average over all pairs of values of controlled factors for each $CH_v$ variant. \rev{See Appendix H for high-resolution images.} }
    \label{fig:val_results}
    \vspace{-2mm}
\end{figure}

\textbf{Objectives and design}
$CH_{btwn}$\xspace derives from $CH$ by using tricks T2, T3, and T4. We want to evaluate the role \ma{these tricks play in making IVM$_{btwn}$\xspace satisfy the new axioms. We consider a synthetic dataset from a previous study \cite{abbas19cgf, AupetitVIS19} made of two bivariate Gaussian clusters (class labels) with various levels of CLM, to which we add noisy dimensions.} We consider four variants $CH_v$\xspace of $CH_{btwn}$\xspace with shift (T2) and range (T4) tricks switched \textit{on} or \textit{off} ($CH_v\in\{CH,CH_{T2},CH_{T4},CH_{T2\&T4}=CH_{btwn}\}$). \rev{The effect of T1 is not evaluated as B1 is already satisfied by both $CH$ and $CH_{btwn}$, and the effect of} T3 is not evaluated because \rev{the ground truth synthetic datasets contain only two classes}. We control the cardinality (B1) and dimension (B2) of the datasets to evaluate how sensitive $CH_v$\xspace variants are to \ma{variations of these conditions (the lower, the better)}. \rev{We do not control class-cardinality (B3) as the number of classes (2) is imposed by the available data. Range-invariance (B4) is not controlled as it is imposed by the min-max trick (T4) and not a characteristic of the datasets.}

\textbf{Datasets} We prepared 1,000 base datasets $\{X_1\dots X_{1000}\}$, each one consisting of $|X|=10,000$ points sampled from two Gaussian clusters ($|C|=2$) within the 2D space and augmented with $98$ noisy dimensions ($\Delta=100$). We controlled the eight independent parameters (ip) of the Gaussians: two covariance matrices (3 ip each), class proportions (1 ip), and the distance between Gaussian means (1 ip), following a previous study \cite{abbas19cgf, AupetitVIS19} (see figure in Appendix C). We add Gaussian noise along the supplementary dimensions, to each cluster-generated data, with a mean 0 and a variance equal to the minimum span of that cluster's covariance. We generated any dataset $X_{i,t}$ by specifying a triplet $(X_i,N_t,\Delta_t)$ with $X_i$ a base dataset, $N_t$ the number of data randomly sampled from $X_i$ preserving cluster proportions, and $\Delta_t$ its dimension where the first two dimensions always correspond to the 2D cluster space.
\textbf{Sensitivity to data-cardinality (B1)} (\autoref{fig:val_results} top)
For each of the $1000$ base data $X_i$, we generated $11$ datasets $X_{i,t}=(X_i,N_t,\Delta_t)_{i\in[1\dots1000],t\in[0\dots10]}$ with the controlled data cardinality set to $N_t=500t+5000$ and $\Delta_t$ drawn uniformly at random from $[2,\dots ,100]$.
\textbf{Sensitivity to dimensionality (B2)} (\autoref{fig:val_results} bottom)
For each of the $1000$ base data $X_i$, we generated $11$ datasets $X_{i,t}=(X_i,N_t,\Delta_t)_{i\in[1\dots1000],t\in[0\dots10]}$ with $N_t$ drawn uniformly at random from $[500,\dots,5000]$ and the controlled dimension set to $\Delta_0=2$ or $\Delta_t=10t, \forall t>0$.

\textbf{Measurements}
For each $CH_v$, we compute the matching between a pair ($a,b)$ of values of the controlled factor $t$ (\textit{e.g.} ($\Delta_a,\Delta_b)=(10,30)$) across all $1000$ base data using: $S_{k\in[1\dots 1000]}(CH_v(C,X_{k,a},d),CH_v(C,X_{k,b},d))$, where S is the Symmetric Mean Absolute Percentage Error (SMAPE) \cite{tofallis15jors} adapted to compare measures with different ranges: ${\displaystyle {S_{k\in K} (F_k,G_k)}={\frac {1}{n}}\sum _{k\in K}\frac{|F_{k}-G_{k}|}{|F_{k}|+|G_{k}|}}$ (0 best, 1 worst).

\textbf{Results}
\autoref{fig:val_results} shows that all $CH$ variants are slightly sensitive to changes in the data size (a--d), with a larger difference of size leading to bigger errors (off-diagonal darker shades of blue). The average error is about $10\%$ for all variants except $CH_{btwn}$\xspace (top row, blue, orange, and green bars), and $CH_{btwn}$\xspace is five times less sensitive to data cardinality than any other variant (top row, red bar). Regarding dimensionality (e--h), all variants except $CH_{btwn}$\xspace (h) are more strongly affected by larger differences in dimension, with about $35\%$ error on average, while $CH_{btwn}$\xspace (red bar) is slightly below $20\%$ on average, a two-fold improvement over other variants.

The bar chart shows that the combination of both shift invariance (T2) and range invariance (T4) tricks is necessary to get $CH_{btwn}$\xspace satisfying axioms B1 (cardinality invariance) and B2 (shift invariance). It is unexpected, though, that using the shift invariance trick alone makes $CH_v$ more sensitive to the dimension. However, this can be explained by the fact that the exponential trick cancels the global shift of all distances (what it is designed for), disregarding the effect on the range of the IVM itself (a non-linear aggregation of distances), a factor that is then mitigated by the range trick (T4).

\subsection{Between-dataset Rank Correlation Analysis}

\begin{table}[t]
    \centering
    \scalebox{0.77}{
    \begin{tabular}{crrrrr}
    \toprule

    \ma{Spearman's rank correlation}& &\multicolumn{4}{c}{\ma{GT-ranking EVMs}}  \\
    \ma{with approximate ground truth CLM}& & \texttt{ami} & \texttt{arand} & \texttt{vm} & \rev{\texttt{nmi}} \\
    \midrule
    \multirowcell{8}{Classifiers } & SVM & 0.5427 & 0.6235 & 0.4625 & 0.4827\\
                                 & $k$NN & 0.4876 & 0.5810 & 0.3974 & 0.4094\\
                                 & MLP & 0.4405 & 0.5386 & 0.3600  & 0.3761\\
                                 & NB & 0.4126 & 0.5276 & 0.3157  & 0.3130\\
                                 & RF & 0.4893 & 0.5741 & 0.3991 & 0.3889\\
                                 & LR & 0.4456 & 0.5382 & 0.3666  & 0.3873\\
                                 & LDA & 0.4999 &  0.5726 & 0.3945 & 0.3606\\
                                 & Ensemble & 0.5922 & 0.6748 & 0.4614  & 0.4099\\
    \midrule
         \multirowcell{6}{IVM$_{wthn}$\xspace}
          & \rev{Silhouette} & 0.5648 & 0.6800 & 0.4549 &   0.4208\\
          & \rev{Xie-Beni}   & 0.6201 & $^{*}$0.7019 & 0.4934 & 0.4446\\
          & Dunn             & 0.4026    & 0.3534 & 0.5366 & $^{*}$0.5979\\
          & I Index          & 0.5668  & 0.5957 & $^{**}$0.6086 &  $^{**}$0.6454\\
          & Davies-Bouldin   & $^{**}$0.7091 & $^{**}$0.7513 &  $^{*}$0.5719 & 0.5015\\
          & $CH$             & $^{*}$0.5923& 0.6222 & 0.4487 & 0.3810\\
    \midrule
           IVM$_{btwn}$\xspace (ours)  & $CH_{btwn}$   & $^{***}$0.7893& $^{***}$0.7981 & $^{***}$0.7022  & $^{***}$0.6561 \\

    \bottomrule
    \addlinespace[0.115cm]
    \multicolumn{5}{l}{
        \footnotesize
        $^{***}$, $^{**}$, $^{*}$: first, second, and third highest scores \ma{for each EVM}
    } \\
    \multicolumn{5}{l}{
        \footnotesize
        Every result was validated to be statistically significant ($p < .001)$ \rev{through Spearman's rank correlation test.}
    }
    \end{tabular}
    }
    \vspace{2.5mm}
    \caption{
    Rank correlations between \rev{approximate} ground truth CLM ranking \ma{based on 9 clustering techniques} and estimated CLM ranking obtained by $CH_{btwn}$\xspace, various IVM$_{wthn}$\xspace, and classifiers.
    $CH_{btwn}$ rankings (***) outperform \rev{all the competitors and achieved an improvement of about 20\% compared to its within-dataset version ($CH$).}  }
    \label{tab:dataset_entire}
    \vspace{-3mm}
\end{table}

\label{sec:bdsa}

\textbf{Objectives and design}
We  assess $CH_{btwn}$ against competitors for best estimating the CLM ranking of publicly available labeled datasets.
We \rev{approximate} a ground truth CLM quality for each labeled dataset using multiple clustering techniques. We then compare the rankings made by all competitors and $CH_{btwn}$ to this ground truth using Spearman's rank correlation.

\textbf{Datasets} We collected 96 publicly available labeled datasets with diverse numbers of data points\rev{,} class labels, cluster patterns (presumably), and dimensionality (Appendix E).

\textbf{\rev{Approximating the ground truth} CLM} \rev{For lack of definite ground truth clusters in multidimensional real data,} we used the maximum EVM score achievable by \ma{nine} various clustering techniques on a labeled dataset as \rev{an approximation of the} ground truth (GT) CLM score for that dataset. These GT scores were used to get the GT-ranking of all the datasets.
This scheme relies on the fact that high EVM implies good CLM (\autoref{sec:intro}; \autoref{fig:clm} A and D).
We used Bayesian optimization \cite{snoek12neurips} to find the best hyperparameter setting for each clustering technique.
We obtained GT-ranking based on \ma{four} EVMs: adjusted rand index (\texttt{arand}) \cite{santos09icann}, adjusted mutual information (\texttt{ami}) \cite{vinh10jmlr}, V-measure (\texttt{vm}) \cite{rosenberg07emnlp}, \rev{and normalized mutual information (\texttt{nmi}) \cite{strehl02jmlr} with geometric mean}.
For clustering techniques, we used HDBSCAN \cite{campello13akddm}, DBSCAN \cite{schubert17tds},  $K$-Means \cite{hartigan79jstor}, $K$-Medoids  \cite{park09esa}, $X$-Means \cite{pelleg00icml}, Birch \cite{zhang96sigmod}, and Single, Average, and Complete variants of Agglomerative Clustering \cite{mullner11arxiv} (Appendix D).

\textbf{Competitors} We compared supervised classifiers, IVM$_{wthn}$\xspace,  and  $CH_{btwn}$ to the GT ranking. For classifiers, we used SVM, $k$NN, Multilayer Perceptron (MLP), Naive Bayesian Networks (NB), Random Forest (RF), Logistic Regression (LR), Linear Discriminant Analysis (LDA), and their ensembles; \rev{the selected classifiers are the ones used for evaluating clustering in  Rodr\'iguez et al. }\cite{rodriques18kbs}. We measured the classification score of a given labeled dataset, using five-fold cross validation and Bayesian optimization \cite{snoek12neurips} to find the best hyperparameter setting. The accuracy in predicting class labels was averaged over the five validation sets to get a proxy of the CLM score for that dataset. For the ensemble method, we got the proxy as the highest accuracy score among the seven classifiers for each dataset independently. \rev{Regarding IVM$_{wthn}$\xspace, we considered the list of Liu et al. \cite{liu10idcm}, except the ones optimized based on the elbow rule (\textit{e.g.}, Modified Hubert $\Gamma$ statistic \cite{hubert85classification})
and the ones requiring several clustering results (\textit{e.g.}, S\_Dbw index \cite{halkidi01icdm}), thus we used:} $CH$\xspace, Davies-Bouldin index \cite{davies79tpami}, Dunn index \cite{dunn74joc}, I index \cite{maulik02tpami}, Silhouette \cite{rousseeuw87jcam}, and Xie-Beni index \cite{xie91tpami} (See details in Appendix D).

\textbf{Results}
\autoref{tab:dataset_entire} shows that for every EVM, $CH_{btwn}$ (***) outperforms all \rev{competitors. Especially, $CH_{btwn}$ achieved a performance improvement of about 20\% compared to $CH$. The second (**) and third (*) places  vary depending on the EVM, but they are all part of the IVM$_{wthn}$\xspace category.}
Therefore, $CH_{btwn}$ can be used as a reliable measure \ma{of CLM to rank datasets} (\autoref{fig:btw_result}) despite their drastic variations in terms of dimension, number of class labels, and data size. It also runs far faster  than optimizing any of the GT clustering techniques (tens of seconds versus several hours for all 96 datasets; Appendix F), clearly demonstrating its benefit \ma{both in terms of time and accuracy}.

\section{Application: Ranking the Labeled Datasets for Reliable EVM}

\label{sec:selection}

\begin{figure}[t]
    \centering
    \includegraphics[width=\textwidth]{figures/between.pdf}
    \vspace{-3mm}
    \caption{

    All (gray points) and best (red points) \texttt{ami} scores of GT clustering techniques for the $96$ benchmark datasets, ranked by $CH$\xspace (left) and $CH_{btwn}$\xspace (right). \ma{The top $20$ datasets in terms of} $CH_{btwn}$ (right) are \ma{the most reliable to evaluate and compare clustering techniques using EVMs.}

    }
    \label{fig:btw_result}
    \vspace{-2mm}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[width=\textwidth]{figures/rank_analysis.pdf}
        \vspace{-3mm}
    \caption{(a)

     Distribution of pairwise rank stability for \texttt{bottom-20} (blue; $\mathcal{P}^-$), \texttt{full} (orange; $\mathcal{P}^*$), and \texttt{top-20} (green; $\mathcal{P}^+$) datasets. (b)
     Rankings of clustering techniques for each set. All rankings are based on \texttt{ami} EVM averaged over the datasets within each set. Using the datasets top-ranked by $CH_{btwn}$\xspace as a proxy of their good CLM leads to stable and reliable rankings ((a) green bar). }
    \label{fig:rank_analysis}
    \vspace{-2mm}
\end{figure}

\textbf{Objectives and design} We want to demonstrate the importance of \maa{evaluating} the CLM of benchmark datasets prior to conducting the external validation of clustering techniques.
Here, in addition to the \texttt{full} set of $96$ public
datasets ($\mathcal{P}^*$), we consider the \texttt{top-20}  ($\mathcal{P}^+$) and \texttt{bottom-20}  ($\mathcal{P}^-$) datasets as per their $CH_{btwn}$\xspace rank (\autoref{fig:btw_result}b) (the \texttt{top-20} and \texttt{bottom-20} datasets are given in Appendix C).

We consider simulating the situation where a data scientist would arbitrarily choose $10$ benchmark datasets ($\mathcal{B}$) among the datasets at hand for the task T of ranking clustering techniques according to $EVM_{\mathcal{B}}$, the average EVM over $\mathcal{B}$.
For each $\mathcal{P}\in\{\mathcal{P}^+,\mathcal{P}^*,\mathcal{P}^-\}$, we simulate $100$ times picking $\mathcal{B}$ at random among $\mathcal{P}$. For each $\mathcal{P}$, we measure the pairwise rank stability $P_\mathcal{B}(A,B)=\max(1-p,p)$ of clustering techniques A and B over $\mathcal{B}$ by counting the proportion $p$ of cases $\texttt{ami}_{\mathcal{B}}(A)>\texttt{ami}_\mathcal{B}(B)$.

\textbf{Assumptions} We expect that conducting T on any subset of good-CLM datasets would provide similar rankings (\autoref{fig:clm}A) where pairwise ranks remain stable ($\forall (A,B), P_\mathcal{B}(A,B)\approx 1$), whereas conducting T using bad-CLM datasets would lead to arbitrary and unstable rankings ($\forall (A,B), P_\mathcal{B}(A,B)$ spread over $[0.5, 1]$)
(\autoref{fig:clm}BEH).

\textbf{Results and discussion} \autoref{fig:rank_analysis}a shows that pairwise ranks stay stable only in $\mathcal{P}^+$, which verifies our assumptions.
Moreover, we found that the rankings of clustering techniques made by $EVM_{\mathcal{P^+}}$, $EVM_{\mathcal{P^*}}$, and $EVM_{\mathcal{P^-}}$ are completely different (\autoref{fig:rank_analysis}b).
\rev{Still, some datasets within $\mathcal{P}^-$ (\textit{e.g.}, Spambase, Hepatitis \cite{asuncion07uci})
have been used for external clustering validation in previous studies \cite{rehioui16pcs, khan21icecit, monath17nips, monath19kdd}}
\maa{without CLM evaluation, casting doubt on their conclusion and showing this issue shall gain more attention in the benchmarking community.}
 \maa{CLM scores could be used further to inform benchmarking results (Appendix G) or to improve dataset's reliability by modifying datasets' class labels.}

\section{Conclusion}

In this research, we provided a grounded way to \maa{evaluate the reliability of} benchmark labeled datasets used for the external evaluation of clustering techniques. \maa{We proposed to measure their level of cluster-label matching (CLM). }
We presented four between-dataset axioms and  technical tricks to generate measures that satisfy them. We used these tricks to design a new between-dataset internal validation measure $CH_{btwn}$\xspace generalizing the Calinski-Harabasz index for across-datasets comparisons. We studied the accuracy of this measure to rank $96$ benchmark datasets and showed that it outperforms all competitors in terms of time and accuracy. We demonstrated its usefulness in determining the most reliable datasets for comparing clustering techniques.

As future work, we want to explore further the use of our tricks to generalize other IVM$_{wthn}$\xspace, \maa{ and explore how to use the CLM score to build better clustering benchmarks.}

redpajama_set_name: RedPajamaArXiv


If you're thinking about selling your home, we can help you navigate the home selling process, prepare your home for showings, and negotiate the best price for your home. But first, let's determine what your Morgan Park property is worth. Tell us about your home below to get started.
Morgan Park is tucked off of Avent Ferry Road near the NC-55 bypass.
Just around the corner is Southpark Village shopping center, which is anchored by a Harris Teeter.
Morgan Park offers new construction houses with 3 to 6 bedrooms.
Drive down Avent Ferry Road in Holly Springs and you'll see how the town balances its modern growth with historic roots. Mature trees and plowed fields still flank the road, and it doesn't take long to pass established farms mingled with new construction communities. In fact, The Family Farm on Avent Ferry, which is located across the street from Holly Glen, has been a local landmark for over a century.
Morgan Park is the latest addition to this part of Holly Springs. This enclave of new homes, which overlooks a country cottage with its own horse paddock, offers a swimming pool, fitness center, and playground.
Shop for groceries or grab a bite to eat at Southpark Village just 5 minutes away, or spend an afternoon exploring all that Holly Springs Towne Center has to offer.
Holly Springs residents have their pick of recreation programs, parks, and greenway trails. Parrish Womble Park and Bass Lake Park are popular destinations in the area, and they're both just a short drive from Morgan Park.
Getting around town is a breeze with NC-55 just around the corner. From there, it's a short drive to NC-540 (the Triangle Expressway), which can take you to Research Triangle Park and RDU International Airport in about half an hour.
Morgan Park residents can opt to attend schools within the highly acclaimed Wake County Public School System or choose from several private schools and preschools nearby.
Eager to Learn More? Just Give Us a Call!
Whether you have questions about the homes or the area, we have all the information you need. Just give us a call and tell us how we can help.


Pou for Android is the new version of Tamagotchi. You get virtual pet to take care of right on your device, and to dress and decorate however you want. Did you still remember that old game called Tamagoci? It might have been way back in the past, but you must have remembered how fun and addictive it was. You don't have to worry though. You can still play the game even now. It is just that the game has been visually improved, so you can enjoy the game in better way. The game we suggest for that would be Pou. How better is it exactly though? You can find out about that by learning the features it offers. Let's see below then.
Yes, this game has the same gameplay as its first version. In it, you will have to take care of small alien in the form of poop. Just like when you took care of your dog in Tamagoci, you need to pamper him so he can grow up happily. There are all sorts of thing you can do to be able to do such thing to him. You should feed him to fill his belly, play with him so he can be happy, clean him to make him healthy, and give attention to him so he knows you are caring. It is just like how to take care of pets.
This game gives you virtual pet to care about. Caring for him is not all about filling his belly and playing with him only. You can show your affection by dressing him with all sorts of accessories. There are hats, clothes, and even backgrounds for your little virtual pet. Going out with your Android will feel like taking your pet for walk. Don't you think so too? Now, you see Pou is pretty similar to the old game, but we can't deny that its improvement makes it all the more interesting.


Megan Davies- " See You Again/ Love Me Like You nightcore Do/ Sugar" Guitar chords. Loading the chords for ' Nightcore - Chandelier'. Fm Db Ab Eb] Chords for Nightcore - Faded sheet with capo transposer play along with guitar, chandelier piano ukulele & mandolin. Print download in PDF MIDI Chandelier - Sia. Contains printable sheet music plus an interactive, downloadable digital sheet music file. Nightcore - Never Know.
Nightcore chandelier piano sheet. Beginner Piano Guitar Chords/ Lyrics Easy Piano Ukulele chandelier Big Note Piano Guitar Tab Lead Sheet / Fake Book Easy Guitar Tab Guitar Tab ( Single Guitar) Guitar Lead Sheet Flute Duet Piano & Vocal Alto Sax Duet Clarinet Duet Violin chandelier Duet Trumpet Duet chandelier Trombone Duet Cello Duet Guitar Ensemble HDPiano Accordion Super Easy Piano. Print this page For offline use Download PDF Chord sheet. Download Chandelier Sheet Music Sia Pdf free. Nightcore Edition ( sped up). Virtual Piano is fast becoming a form of expression piano communication between different cultures , space , regions of the world – crossing language time.
or piano with interactive chords and diagrams. Well, I don' chandelier t play play piano so the second. Chandelier Sheet Music is a nice song by singer- songwriter Sia australina it was posted on March 17 included in sheet his 1000 album Forms nightcore of Fear. Includes transpose changing speed chandelier , capo hints much more. Chords chandelier for Nightcore - Chandelier. Chandelier ( Piano Version) Piano Accompaniment / Cover Instrumental Karaoke. Animated Overview. I don' t own anything. Our vision is to nightcore spread the joy of playing the piano to every corner of the globe.
Contains printable sheet music plus an interactive, downloadable digital sheet music file. Does not contain lyrics This product is available worldwide Available at a discount in the digital sheet music collection: Titanium - C Instrument & Piano. Music: The Legend of Zelda Skyward Sword Ballad of the godness/ The Legend of Zelda The Wind Waker Title Theme ( Nintendo) I hope you like it, enjoy! All Songs Numb Linkin Park Dubstep Kinds Of Music Trance Good Music Sheet Music.
" Nightcore - Chandelier so that' s what' s this song is called. Worst Day Of My Life Nightcore Chandelier Coco Jones Sia Cover Srod Almenara Diego Of Fear Album Bus Hd Finally 10 15 And Songsters Piano Music Sheet Fall Out Boy.
Shatter Me - Violin, Voice & Piano digital sheet music.


text:
Mayor Park Won-soon…
Mayor Park Won-soon Visited the "Old Korean Legation in Washington D. C.," a Symbol of Independent Diplomacy
Registration date 01/16/2020 WriterSMG views264
On January 12 (Sun.), 2020, during a visit to Washington D. C., the last city he visited in the U. S. during the tour, Seoul Mayor Park Won-soon visited the "Old Korean Legation in Washington D.C.," the symbolic space for the independence of diplomacy in the last period of Joseon.
The Old Korean Legation in Washington D. C. is the first legation that was established in a Western country in the history of Korea, in February 1889. It was used as a Korean legation in the U. S. for 16 years. It is regarded as the milestone of the beginning of the independent diplomacy during the late Joseon period that aimed to expand its diplomatic horizon. The building is more meaningful in that it is the only 19-century legation in Washington D.C. that remains intact.
Kathleen Stephens, the director of the Korea Economic Institute, accompanied Mayor Park during the mayor's visit to the Old Korean Legation, sharing a conversation while looking around the historic space that represents the amicable diplomatic relationship between Korea and the United States. As a distinguished pro-Korean figure, Kathleen Stephens served as the former American Ambassador to Korea.
Seoul Announces "2020 Changing Life in Seoul"
Seoul to host the 18th World Summit of Nobel Peace…
2020 New Year's Address
Seoul Attracts USD 230 Million Investment from Four Promising Silicon Valley Companies
Leading Practice of Sharing and Donation by Changsin-dong, the First Village That Underwent Urban Regeneration

redpajama_set_name: RedPajamaCommonCrawl


A seismoelectric recording system. Consist of seismoeletric signal data pre-adjusters and computer control. A military grade computer used as the control with built in GPS.
In the general survey, unit obtains both the seismic signal, and the combined seismoelectric signals. The seismoelectric survey monitor receiver is used in a civil engineering ground water related survey.
Accessories include: Seismoelectric electrodes, geophones, cable, connectors, etc.
The nature of generation of the Seismic source is critical to the seismic and seismoelecric survey results.
The sledge hammer is popular for most shallow seismic refraction surveys using a multichannel seismograph, but the range of impact is limited. The "buffalo gun" has a wider and deeper impact range of acoustic energy and has been used in the field with great success with the seismoelectric method.
after data processing the Penetration depth of seismoelectric will increase dramatic Hugh.
εxx (e2) is the displacement in the x direction along the x axis at electrode 2, εzz (e1) is the displacement in the z direction along the z axis at electrode 1 and εzz (e2) is the displacement in the z direction along the z axis at electrode 2. Partial derivatives of the displacement with respect to time are included, as are the same variables used in Equation 1.
Use of these equations allows us to analyze a streaming potential signal in respect to time and arrive at any two dimensional space. Furthermore, the equivalent sets of equations for three dimensions can obviously be derived. It is also possible to reduce them back to the one- dimensional versions and use that for some surveying.
The main problem of buffalo gun is its loud sound blast, which may restrict its use in populated area. We have developed a soundless device in the buffalo gun, and it works very well with the same seismic impact by using a 12 gauge shot gun cartrid.
In order to meet some special requirements of our clients, we have also tested weight drop seismic source methodology of generation in the seismoelectric survey.
The tested data shown the one drop energy of A200 P/S Hammer is similar to a 2.5 inch 12 gauge cartridge acoustic impulse. However, the relative cost of this weight drop seismic source is much more than using a buffalo gun.
To mathematically derive expressions of seismoelectric effect propagation, an important assumption is made that seismoelectric signals "propagate" at the same velocity as its inducing seismic wave. This assumption is validated through its similarity to headlight traces observed on a highway. A travelling car and its taillight represent the seismic wave and the seismoelectric signal it generates, respectively. As light speed is obviously faster than car speed, we can observe the movement of a car by tracking its tailing light in a dark night. Similarly, electric current transmits much faster than seismic waves in rocks. The transit time of electric current from the affected rock to a detector on the ground is omitted. By honoring our assumption, one can plot the propagation of seismoelectric effect the same way as of its inducing seismic wave.
where E(t) is streaming potential, ω is the frequency of the seismic wave, Δρ is the density difference between the rock and the water, P1 and P2 are rock stresses of the seismic wave at electrodes 1 and 2, εis the permittivity of dielectric of the fluid, ζ is the zeta potential, η is the viscosity and σ is the fluid conductivity. π is pai(3.1415926).


text:
The Role of Heat Shock Protein (HSP) in Atherosclerosis: Pathophysiology and Clinical Opportunities
Author(s): X. Lu, V. Kakkar
Heat Shock Proteins; An Overview
Journal: Current Pharmaceutical Biotechnology
Author(s): Lutfi Tutar, Yusuf Tutar
Heat Shock Proteins And Neuroprotection
Journal: Recent Patents on DNA & Gene Sequences (DIscontinued)
Author(s): Seema Trivedi
Heat Shock Proteins as Prognostic Markers of Cancer
Author(s): Evgeniya V. Kaigorodova, Mariya V. Bogatyuk
Thermal Adaptation of Heat Shock Proteins
Journal: Current Protein & Peptide Science
Author(s): A. Muga, F. Moro
Heat Shock Protein (HSP) Drug Discovery and Development: Targeting Heat Shock Proteins in Disease
Journal: Current Topics in Medicinal Chemistry
Author(s): Liza Shrestha, Alexander Bolaender, Hardik J. Patel, Tony Taldone
Heat Shock Proteins Protect Against Ischemia and Inflammation Through Multiple Mechanisms
Journal: Inflammation & Allergy - Drug Targets (Discontinued)
Author(s): Quinton Jones, Tracy S. Voegeli, Gefeng Li, Yu Chen, R. William Currie
Role of Heat Shock Proteins in Atherosclerosis and Atrial Fibrillation
Journal: Current Immunology Reviews
Author(s): Sophie Z. Lin,Todd C. Crawford,Kaushik Mandal
Heat Shock Proteins in Diabetes and Wound Healing
Author(s): Mustafa Atalay, Niku Oksala, Jani Lappalainen, David E. Laaksonen, Chandan K. Sen, Sashwati Roy
Go To 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 Results 1 - 1 of 1291


Dr. Ken Nordlund and Courtney Halbach will lead participants through hands-on exercises to learn how to use the Positive Pressure Tube Calculator (PPTC) to design positive pressure tube ventilation (PPTV) systems for calf barns. Participants will leave this workshop with field-ready skills to improve respiratory health in calf barns. Trainees will be listed on The Dairyland Initiative website as trained consultants in positive pressure tube design. Following the workshop, trainees will have the opportunity to become a Dairyland Initiative Certified Consultant in PPTV design for young stock housing. Participants must bring a computer that has Microsoft Office installed.