def multi_class_classification(data_X,data_Y):
    '''
    calculate multi-class classification and return related evaluation metrics
    '''

    svc = svm.SVC(C=1, kernel='linear')
    # X_train, X_test, y_train, y_test = train_test_split( data_X, data_Y, test_size=0.4, random_state=0) 
    clf = svc.fit(data_X, data_Y) #svm
    # array = svc.coef_
    # print array
    predicted = cross_val_predict(clf, data_X, data_Y, cv=2)
    print "accuracy",metrics.accuracy_score(data_Y, predicted)
    print "f1 score macro",metrics.f1_score(data_Y, predicted, average='macro') 
    print "f1 score micro",metrics.f1_score(data_Y, predicted, average='micro') 
    print "precision score",metrics.precision_score(data_Y, predicted, average='macro') 
    print "recall score",metrics.recall_score(data_Y, predicted, average='macro') 
    print "hamming_loss",metrics.hamming_loss(data_Y, predicted)
    print "classification_report", metrics.classification_report(data_Y, predicted)
    print "jaccard_similarity_score", metrics.jaccard_similarity_score(data_Y, predicted)
    # print "log_loss", metrics.log_loss(data_Y, predicted)
    print "zero_one_loss", metrics.zero_one_loss(data_Y, predicted)
    # print "AUC&ROC",metrics.roc_auc_score(data_Y, predicted)
    # print "matthews_corrcoef", metrics.matthews_corrcoef(data_Y, predicted) 

def evaluation_analysis(true_label,predicted): 
    '''
    return all metrics results
    '''
    print "accuracy",metrics.accuracy_score(true_label, predicted)
    print "f1 score macro",metrics.f1_score(true_label, predicted, average='macro')     
    print "f1 score micro",metrics.f1_score(true_label, predicted, average='micro') 
    print "precision score",metrics.precision_score(true_label, predicted, average='macro') 
    print "recall score",metrics.recall_score(true_label, predicted, average='macro') 
    print "hamming_loss",metrics.hamming_loss(true_label, predicted)
    print "classification_report", metrics.classification_report(true_label, predicted)
    print "jaccard_similarity_score", metrics.jaccard_similarity_score(true_label, predicted)
    print "log_loss", metrics.log_loss(true_label, predicted)
    print "zero_one_loss", metrics.zero_one_loss(true_label, predicted)
    print "AUC&ROC",metrics.roc_auc_score(true_label, predicted)
    print "matthews_corrcoef", metrics.matthews_corrcoef(true_label, predicted) 

def matthews_correlation_coefficient_scorer(
    golds: ndarray,
    probs: Optional[ndarray],
    preds: ndarray,
    uids: Optional[List[str]] = None,
) -> Dict[str, float]:
    """Matthews correlation coefficient (MCC).

    Args:
      golds: Ground truth values.
      probs: Predicted probabilities.
      preds: Predicted values.
      uids: Unique ids, defaults to None.

    Returns:
      Matthews correlation coefficient score.
    """
    # Convert probabilistic label to hard label
    if len(golds.shape) == 2:
        golds = prob_to_pred(golds)

    return {"matthews_corrcoef": matthews_corrcoef(golds, preds)} 

def test_confusion_matrix_binary():
    # Test confusion matrix - binary classification case
    y_true, y_pred, _ = make_prediction(binary=True)

    def test(y_true, y_pred):
        cm = confusion_matrix(y_true, y_pred)
        assert_array_equal(cm, [[22, 3], [8, 17]])

        tp, fp, fn, tn = cm.flatten()
        num = (tp * tn - fp * fn)
        den = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))

        true_mcc = 0 if den == 0 else num / den
        mcc = matthews_corrcoef(y_true, y_pred)
        assert_array_almost_equal(mcc, true_mcc, decimal=2)
        assert_array_almost_equal(mcc, 0.57, decimal=2)

    test(y_true, y_pred)
    test([str(y) for y in y_true],
         [str(y) for y in y_pred]) 

def compute_metrics(task_name, preds, labels):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    else:
        raise KeyError(task_name) 

def compute_metrics(task_name, preds, labels):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    else:
        raise KeyError(task_name) 

def compute_metrics(task_name, pred_srs, label_srs):
    assert len(pred_srs) == len(label_srs)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(label_srs, pred_srs)}
    elif task_name == "sst":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "mrpc":
        return acc_and_f1(pred_srs, label_srs)
    elif task_name == "stsb":
        return pearson_and_spearman(pred_srs, label_srs)
    elif task_name == "qqp":
        return acc_and_f1(pred_srs, label_srs)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(pred_srs, label_srs)}
    else:
        raise KeyError(task_name) 

def score_eval_func(self, y_true, y_pred, mode='accuracy'):
        prob_pred = two_class_encoding(y_pred)
        label_pred = np.argmax(prob_pred, axis=1)
        if mode == 'accuracy':
            score_pred = accuracy_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'precision':
            score_pred = precision_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'recall':
            score_pred = recall_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'f1':
            score_pred = f1_score(y_true=y_true, y_pred=label_pred)
        elif mode == 'MCC':
            score_pred = matthews_corrcoef(y_true=y_true, y_pred=label_pred)
        else:
            raise ValueError('Score function mode unrecognized! Must from one in the list '
                             '[\'accuracy\', \'precision\',\'recall\',\'f1\',\'MCC\']')

        return score_pred 

def validation_step(self, inputs, model: tf.keras.Model, metrics=None):
    if self.metric_type == 'accuracy':
      return super(SentencePredictionTask,
                   self).validation_step(inputs, model, metrics)
    features, labels = inputs
    outputs = self.inference_step(features, model)
    loss = self.build_losses(
        labels=labels, model_outputs=outputs, aux_losses=model.losses)
    logs = {self.loss: loss}
    if self.metric_type == 'matthews_corrcoef':
      logs.update({
          'sentence_prediction':
              tf.expand_dims(
                  tf.math.argmax(outputs['sentence_prediction'], axis=1),
                  axis=0),
          'labels':
              labels,
      })
    if self.metric_type == 'pearson_spearman_corr':
      logs.update({
          'sentence_prediction': outputs['sentence_prediction'],
          'labels': labels,
      })
    return logs 

def get_metric_fn(task_name):
    if task_name == "cola":
        return lambda p, l: {"mcc": matthews_corrcoef(p, l)}
    if task_name == "sst-2":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "mrpc":
        return acc_and_f1
    if task_name == "sts-b":
        return pearson_and_spearman
    if task_name == "qqp":
        return acc_and_f1
    if task_name == "mnli":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "mnli-mm":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "qnli":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "rte":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    if task_name == "wnli":
        return lambda p, l: {"acc": simple_accuracy(p, l)}
    raise KeyError(task_name) 

def test_confusion_matrix_binary():
    # Test confusion matrix - binary classification case
    y_true, y_pred, _ = make_prediction(binary=True)

    def test(y_true, y_pred):
        cm = confusion_matrix(y_true, y_pred)
        assert_array_equal(cm, [[22, 3], [8, 17]])

        tp, fp, fn, tn = cm.flatten()
        num = (tp * tn - fp * fn)
        den = np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))

        true_mcc = 0 if den == 0 else num / den
        mcc = matthews_corrcoef(y_true, y_pred)
        assert_array_almost_equal(mcc, true_mcc, decimal=2)
        assert_array_almost_equal(mcc, 0.57, decimal=2)

    test(y_true, y_pred)
    test([str(y) for y in y_true],
         [str(y) for y in y_pred]) 

def test_matthews_corrcoef_nan():
    assert_equal(matthews_corrcoef([0], [1]), 0.0)
    assert_equal(matthews_corrcoef([0, 0], [0, 1]), 0.0) 

def test_matthews_corrcoef_against_numpy_corrcoef():
    rng = np.random.RandomState(0)
    y_true = rng.randint(0, 2, size=20)
    y_pred = rng.randint(0, 2, size=20)

    assert_almost_equal(matthews_corrcoef(y_true, y_pred),
                        np.corrcoef(y_true, y_pred)[0, 1], 10) 

def test_matthews_corrcoef_against_jurman():
    # Check that the multiclass matthews_corrcoef agrees with the definition
    # presented in Jurman, Riccadonna, Furlanello, (2012). A Comparison of MCC
    # and CEN Error Measures in MultiClass Prediction
    rng = np.random.RandomState(0)
    y_true = rng.randint(0, 2, size=20)
    y_pred = rng.randint(0, 2, size=20)
    sample_weight = rng.rand(20)

    C = confusion_matrix(y_true, y_pred, sample_weight=sample_weight)
    N = len(C)
    cov_ytyp = sum([
        C[k, k] * C[m, l] - C[l, k] * C[k, m]
        for k in range(N) for m in range(N) for l in range(N)
    ])
    cov_ytyt = sum([
        C[:, k].sum() *
        np.sum([C[g, f] for f in range(N) for g in range(N) if f != k])
        for k in range(N)
    ])
    cov_ypyp = np.sum([
        C[k, :].sum() *
        np.sum([C[f, g] for f in range(N) for g in range(N) if f != k])
        for k in range(N)
    ])
    mcc_jurman = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)
    mcc_ours = matthews_corrcoef(y_true, y_pred, sample_weight)

    assert_almost_equal(mcc_ours, mcc_jurman, 10) 

def test_matthews_corrcoef():
    rng = np.random.RandomState(0)
    y_true = ["a" if i == 0 else "b" for i in rng.randint(0, 2, size=20)]

    # corrcoef of same vectors must be 1
    assert_almost_equal(matthews_corrcoef(y_true, y_true), 1.0)

    # corrcoef, when the two vectors are opposites of each other, should be -1
    y_true_inv = ["b" if i == "a" else "a" for i in y_true]
    assert_almost_equal(matthews_corrcoef(y_true, y_true_inv), -1)

    y_true_inv2 = label_binarize(y_true, ["a", "b"])
    y_true_inv2 = np.where(y_true_inv2, 'a', 'b')
    assert_almost_equal(matthews_corrcoef(y_true, y_true_inv2), -1)

    # For the zero vector case, the corrcoef cannot be calculated and should
    # result in a RuntimeWarning
    mcc = assert_warns_div0(matthews_corrcoef, [0, 0, 0, 0], [0, 0, 0, 0])

    # But will output 0
    assert_almost_equal(mcc, 0.)

    # And also for any other vector with 0 variance
    mcc = assert_warns_div0(matthews_corrcoef, y_true, ['a'] * len(y_true))

    # But will output 0
    assert_almost_equal(mcc, 0.)

    # These two vectors have 0 correlation and hence mcc should be 0
    y_1 = [1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1]
    y_2 = [1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1]
    assert_almost_equal(matthews_corrcoef(y_1, y_2), 0.)

    # Check that sample weight is able to selectively exclude
    mask = [1] * 10 + [0] * 10
    # Now the first half of the vector elements are alone given a weight of 1
    # and hence the mcc will not be a perfect 0 as in the previous case
    assert_raises(AssertionError, assert_almost_equal,
                  matthews_corrcoef(y_1, y_2, sample_weight=mask), 0.) 

def test_matthews_corrcoef_overflow(n_points):
    # https://github.com/scikit-learn/scikit-learn/issues/9622
    rng = np.random.RandomState(20170906)

    def mcc_safe(y_true, y_pred):
        conf_matrix = confusion_matrix(y_true, y_pred)
        true_pos = conf_matrix[1, 1]
        false_pos = conf_matrix[1, 0]
        false_neg = conf_matrix[0, 1]
        n_points = len(y_true)
        pos_rate = (true_pos + false_neg) / n_points
        activity = (true_pos + false_pos) / n_points
        mcc_numerator = true_pos / n_points - pos_rate * activity
        mcc_denominator = activity * pos_rate * (1 - activity) * (1 - pos_rate)
        return mcc_numerator / np.sqrt(mcc_denominator)

    def random_ys(n_points):    # binary
        x_true = rng.random_sample(n_points)
        x_pred = x_true + 0.2 * (rng.random_sample(n_points) - 0.5)
        y_true = (x_true > 0.5)
        y_pred = (x_pred > 0.5)
        return y_true, y_pred

    arr = np.repeat([0., 1.], n_points)  # binary
    assert_almost_equal(matthews_corrcoef(arr, arr), 1.0)
    arr = np.repeat([0., 1., 2.], n_points)  # multiclass
    assert_almost_equal(matthews_corrcoef(arr, arr), 1.0)

    y_true, y_pred = random_ys(n_points)
    assert_almost_equal(matthews_corrcoef(y_true, y_true), 1.0)
    assert_almost_equal(matthews_corrcoef(y_true, y_pred),
                        mcc_safe(y_true, y_pred)) 

def compute_mcc(predicts, labels):
    return 100.0 * matthews_corrcoef(labels, predicts) 

def phi_coef(self):
        return sm.matthews_corrcoef(self.labels, self.predictions) 

def glue_compute_metrics(task_name, preds, labels):
        assert len(preds) == len(labels)
        if task_name == "cola":
            return {"mcc": matthews_corrcoef(labels, preds)}
        elif task_name == "sst-2":
            return {"acc": simple_accuracy(preds, labels)}
        elif task_name == "mrpc":
            return acc_and_f1(preds, labels)
        elif task_name == "sts-b":
            return pearson_and_spearman(preds, labels)
        elif task_name == "qqp":
            return acc_and_f1(preds, labels)
        elif task_name == "mnli":
            return {"acc": simple_accuracy(preds, labels)}
        elif task_name == "mnli-mm":
            return {"acc": simple_accuracy(preds, labels)}
        elif task_name == "qnli":
            return {"acc": simple_accuracy(preds, labels)}
        elif task_name == "rte":
            return {"acc": simple_accuracy(preds, labels)}
        elif task_name == "wnli":
            return {"acc": simple_accuracy(preds, labels)}
        elif task_name == "hans":
            return {"acc": simple_accuracy(preds, labels)}
        else:
            raise KeyError(task_name) 

def compute_metrics(task_name, preds, labels):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "lcqmc":
        return {"acc": simple_accuracy(preds, labels)}
    else:
        raise KeyError(task_name) 

def mcc(y_true, y_pred, round=True):
    """Matthews correlation coefficient
    """
    y_true, y_pred = _mask_value_nan(y_true, y_pred)
    if round:
        y_true = np.round(y_true)
        y_pred = np.round(y_pred)
    return skm.matthews_corrcoef(y_true, y_pred) 

def mcc(preds, labels):
    return {"mcc": matthews_corrcoef(labels, preds)} 

def compute_metrics(task_name, preds, labels, sentences=None, error_file=None):
    assert len(preds) == len(labels)
    if task_name == "cola":
        return {"mcc": matthews_corrcoef(labels, preds)}
    elif task_name == "sst-2":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mrpc":
        return acc_and_f1(preds, labels)
    elif task_name == "sts-b":
        return pearson_and_spearman(preds, labels)
    elif task_name == "qqp":
        return acc_and_f1(preds, labels)
    elif task_name == "mnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "mnli-mm":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "qnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "rte":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "wnli":
        return {"acc": simple_accuracy(preds, labels)}
    elif task_name == "semeval2014-atsc":
        return acc_and_f1macro(preds, labels)
    else:
        raise KeyError(task_name) 

def mcc(y, z, round=True):
    """Compute Matthew's correlation coefficient."""
    if round:
        y = np.round(y)
        z = np.round(z)
    return skm.matthews_corrcoef(y, z) 

def matthews_corr(preds, labels):
    return {
        "matthews_corr": matthews_corrcoef(labels, preds),
    } 

def test_matthews_corrcoef(self):
        if not pdml.compat._SKLEARN_ge_019:
            with pytest.raises(ValueError, match='multiclass is not supported'):
                self.df.metrics.matthews_corrcoef() 

def test_matthews_corrcoef(self):
        result = self.df.metrics.matthews_corrcoef()
        expected = metrics.matthews_corrcoef(self.target, self.pred)
        self.assertEqual(result, expected) 

def test_matthews_corrcoef(self):
        msg = 'multilabel-indicator is not supported'
        with pytest.raises(ValueError, match=msg):
            self.df.metrics.matthews_corrcoef() 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc 

def compute_mcc(labels, preds):
    # Compute ROC curve and ROC area for each class
    mcc = matthews_corrcoef(labels.flatten(), preds.flatten())
    return mcc