import numpy as np from scipy.special import softmax class CELoss(object): def compute_bin_boundaries(self, probabilities = np.array([])): #uniform bin spacing if probabilities.size == 0: bin_boundaries = np.linspace(0, 1, self.n_bins + 1) self.bin_lowers = bin_boundaries[:-1] self.bin_uppers = bin_boundaries[1:] else: #size of bins bin_n = int(self.n_data/self.n_bins) bin_boundaries = np.array([]) probabilities_sort = np.sort(probabilities) for i in range(0,self.n_bins): bin_boundaries = np.append(bin_boundaries,probabilities_sort[i*bin_n]) bin_boundaries = np.append(bin_boundaries,1.0) self.bin_lowers = bin_boundaries[:-1] self.bin_uppers = bin_boundaries[1:] def get_probabilities(self, output, labels, logits): #If not probabilities apply softmax! if logits: self.probabilities = softmax(output, axis=1) else: self.probabilities = output self.labels = np.argmax(labels, axis=1) self.confidences = np.max(self.probabilities, axis=1) self.predictions = np.argmax(self.probabilities, axis=1) self.accuracies = np.equal(self.predictions, self.labels) def binary_matrices(self): idx = np.arange(self.n_data) #make matrices of zeros pred_matrix = np.zeros([self.n_data,self.n_class]) label_matrix = np.zeros([self.n_data,self.n_class]) #self.acc_matrix = np.zeros([self.n_data,self.n_class]) pred_matrix[idx,self.predictions] = 1 label_matrix[idx,self.labels] = 1 self.acc_matrix = np.equal(pred_matrix, label_matrix) def compute_bins(self, index = None): self.bin_prop = np.zeros(self.n_bins) self.bin_acc = np.zeros(self.n_bins) self.bin_conf = np.zeros(self.n_bins) self.bin_score = np.zeros(self.n_bins) if index == None: confidences = self.confidences accuracies = self.accuracies else: confidences = self.probabilities[:,index] accuracies = self.acc_matrix[:,index] for i, (bin_lower, bin_upper) in enumerate(zip(self.bin_lowers, self.bin_uppers)): # Calculated |confidence - accuracy| in each bin in_bin = np.greater(confidences,bin_lower.item()) * np.less_equal(confidences,bin_upper.item()) self.bin_prop[i] = np.mean(in_bin) if self.bin_prop[i].item() > 0: self.bin_acc[i] = np.mean(accuracies[in_bin]) self.bin_conf[i] = np.mean(confidences[in_bin]) self.bin_score[i] = np.abs(self.bin_conf[i] - self.bin_acc[i]) class MaxProbCELoss(CELoss): def loss(self, output, labels, n_bins = 15, logits = True): self.n_bins = n_bins super().compute_bin_boundaries() super().get_probabilities(output, labels, logits) super().compute_bins() #http://people.cs.pitt.edu/~milos/research/AAAI_Calibration.pdf class ECELoss(MaxProbCELoss): def loss(self, output, labels, n_bins = 15, logits = True): super().loss(output, labels, n_bins, logits) return np.dot(self.bin_prop,self.bin_score) class MCELoss(MaxProbCELoss): def loss(self, output, labels, n_bins = 15, logits = True): super().loss(output, labels, n_bins, logits) return np.max(self.bin_score) #https://arxiv.org/abs/1905.11001 #Overconfidence Loss (Good in high risk applications where confident but wrong predictions can be especially harmful) class OELoss(MaxProbCELoss): def loss(self, output, labels, n_bins = 15, logits = True): super().loss(output, labels, n_bins, logits) return np.dot(self.bin_prop,self.bin_conf * np.maximum(self.bin_conf-self.bin_acc,np.zeros(self.n_bins))) #https://arxiv.org/abs/1904.01685 class SCELoss(CELoss): def loss(self, output, labels, n_bins = 15, logits = True): sce = 0.0 self.n_bins = n_bins self.n_data = len(output) self.n_class = len(output[0]) super().compute_bin_boundaries() super().get_probabilities(output, labels, logits) super().binary_matrices() for i in range(self.n_class): super().compute_bins(i) sce += np.dot(self.bin_prop,self.bin_score) return sce/self.n_class class TACELoss(CELoss): def loss(self, output, labels, threshold = 0.01, n_bins = 15, logits = True): tace = 0.0 self.n_bins = n_bins self.n_data = len(output) self.n_class = len(output[0]) super().get_probabilities(output, labels, logits) self.probabilities[self.probabilities < threshold] = 0 super().binary_matrices() for i in range(self.n_class): super().compute_bin_boundaries(self.probabilities[:,i]) super().compute_bins(i) tace += np.dot(self.bin_prop,self.bin_score) return tace/self.n_class #create TACELoss with threshold fixed at 0 class ACELoss(TACELoss): def loss(self, output, labels, n_bins = 15, logits = True): return super().loss(output, labels, 0.0 , n_bins, logits)