---
title: "Reproducibility Package for 'A Data Quality Approach to the Identification of Discrimination Risk in Automated Decision Making Systems' _ RQ2 analysis"
author: "Antonio Vetrò, Marco Torchiano, Mariachiara Mecati"
date: 2021-04-12  
output:
 html_document: default
 output_dir: '../results/'
---

This documents contains the reproducibility scripts for the paper

**A Data Quality Approach to the Identification of Discrimination Risk in Automated Decision Making Systems**

by Mariachiara Mecati, Antonio Vetrò, Marco Torchiano

submitted/accepted for publication at Government Information Quarterly.

The goal of our study is to anticipate possible discrimination risks on the basis of the balance features of the training data employed by ADM systems. To this end we formulated two research questions that will drive our investigation:

RQ 1. _How are existing measures able to detect imbalance among the classes of a given attribute in a dataset?_

RQ 2. _Are existing measures able to reveal a discrimination risk when an ADM system is trained with such data?_

In this script we present our code to answer the second Research Question by analyzing data obtained from the investigation in the script Risks_Datasets.Rmd and summarized in unfair-balance.xlsx .

```{r Setup, include=FALSE}
library(ggplot2)
library(tidyverse)
library(ggrepel)
library(latex2exp)
library(readODS)
knitr::opts_chunk$set(echo = FALSE)
```

```{r load, include=FALSE}

run = 'run1'
base_paper = 'base paper measures/'  # base paper measures/  OR 'all attributes' (for no base paper)
model = 'lr'

imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) 
        #slice(10:38) %>% # 29 or 38 (38 -> without Compas, juvenile, community and Age category)
        #filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>%
        #select("Gini","Shannon","Simpson", "IIR",
        #    
        #        "Renyi_2", "Renyi_4", "Renyi_6",
        #        "Renyi_8", "Renyi_10", "Renyi_12",
        #        "Renyi_14", "Renyi_16", "Renyi_18",
        #        "Renyi_20", "Renyi_22", "Renyi_24",
        #        "Renyi_50", "Renyi_100","Renyi_200",
        #        "Renyi_500",
        #    
        #        "Hill_3", "Hill_5", "Hill_7",
        #        "Hill_9", "Hill_11", "Hill_13",
        #        "Hill_15", "Hill_17", "Hill_19",
        #        "Hill_21", "Hill_23", "Hill_25",
        #        "Hill_50", "Hill_100", "Hill_200",
        #        "Hill_500")

# or 'measures smote'                                      # ATTENTION HERE
unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run)
        #slice(10:38) %>%
        #filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>% 
        #select("Diff Ind" ,"Diff TPR","Diff FPR", "Diff PPV","Diff NPV", "Diff OAE")
```


```{r }
round(cor(unfairness_measures,imbalance_measures,method="spearman"), 2) %>%
knitr::kable()
```

```{r Box-plot} 
# base paper analysis
run = 'run1'
base_paper = 'base paper measures/'  # base paper measures/  OR 'all attributes' (for no base paper)
model = 'lr'

imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) %>%
                        select("Dataset -", "Attribute", "Gini","Shannon","Simpson", "IIR")

fairIx = c("Diff Ind" ,"Diff TPR","Diff FPR")       
# or 'measures smote'                                      # ATTENTION HERE
unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run) %>%
                        select("Dataset -", "Attribute", "m",fairIx)  
# "Diff Ind" ,"Diff TPR","Diff FPR"
# "Diff PPV","Diff NPV", "Diff OAE"
        

d <- merge(imbalance_measures, unfairness_measures) %>%
        rename(Dataset=1) %>%
        mutate(Dataset=str_replace(Dataset," *- *","")) %>%
        pivot_longer(starts_with("Diff"),names_to="FairIx", 
                   names_prefix="Diff ",values_to="FairValue")%>%
        transform(FairValue = FairValue*100) %>%
        transform(FairValue = as.integer(FairValue)) %>%
        pivot_longer(c("Gini","Shannon","Simpson","IIR"),names_to="BalIx", 
                   values_to="BalValue") %>%
        transform(BalValue = BalValue*100) %>%
        transform(BalValue = as.integer(BalValue)) %>%
        mutate(FairIx = fct_relevel(fct_relabel(FairIx, ~ 
        if_else(.=="Ind","Independence", paste0("Separation (",.,")"))), "Independence","Separation (TPR)")) %>%
        #if_else(.=="OAE","OAE", paste0("Sufficiency (",.,")"))), "Sufficiency (PPV)","Sufficiency (NPV)")) %>%
        mutate(Balance.Class = fct_rev( cut(BalValue,c(0,33.3,66.7,100),
                                 labels=c("Imbalanced","Undecided","Balanced"),
                                 include.lowest = TRUE, ordered_result = TRUE))) %>%
        mutate(Risk33 = if_else(BalValue < 33,"Higher risk","Lower risk"),
               Risk66 = if_else(BalValue < 66,"Higher risk","Lower risk"))

# Box-Plot

ggplot(d,aes(y=Risk33,x=FairValue,fill=Risk33,color=Risk33))+
  geom_boxplot(alpha=0.5,width=0.6)+
  scale_fill_manual(values = c("Firebrick","gold"),guide=FALSE)+
  scale_color_manual(values = c("Firebrick","gold"),guide=FALSE)+
  ylab("Risk class")+
  xlab("Unfairness index value")+
  facet_grid(BalIx~FairIx)+
  coord_cartesian(clip = "off")+
  annotate("text", x=0,y=0.5,label=TeX("$\\leftarrow$ fair"), 
           parse=TRUE, size=2.5, color="gray40", hjust=0)+
  annotate("text", x=54,y=0.5,label=TeX("unfair $\\rightarrow$"), 
           parse=TRUE, size=2.5, color="gray40",hjust=1)+
#  annotate("text", x=c(0,50),y=c(0,0),label=c("<-- fair","unfair -->"))+
  theme_light()+
  theme(strip.text.y = element_text(angle=0),
        axis.title.y = element_text(angle=0,hjust=0,vjust=1.05,))


#ggsave('images/run1 unfairness distrib. risk 33 lr, baseline, base paper.png', plot = last_plot())
```

```{r models Box-plot} 
# base paper analysis
run = 'run1'
base_paper = 'base paper measures/'  # base paper measures/  OR 'all attributes' (for no base paper)
#model = 'lr'

# Compas and Juvenile removal
model_df <- function(model) {
imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) %>%
                        slice(9:29) %>%  #not compas and juvenile
                        select("Dataset -", "Attribute", "Gini","Shannon","Simpson", "IIR")

fairIx = c("Diff Ind" ,"Diff TPR","Diff FPR")       
# or 'measures smote'                                      # ATTENTION HERE
unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(9:29) %>%
                        select("Dataset -", "Attribute", "m",fairIx)  
# "Diff Ind" ,"Diff TPR","Diff FPR"
# "Diff PPV","Diff NPV", "Diff OAE"
        
d <- merge(imbalance_measures, unfairness_measures) %>%
        rename(Dataset=1) %>%
        mutate(Dataset=str_replace(Dataset," *- *","")) %>%
        pivot_longer(starts_with("Diff"),names_to="FairIx", 
                   names_prefix="Diff ",values_to="FairValue")%>%
        transform(FairValue = FairValue*100) %>%
        transform(FairValue = as.integer(FairValue)) %>%
        pivot_longer(c("Gini","Shannon","Simpson","IIR"),names_to="BalIx", 
                   values_to="BalValue") %>%
        transform(BalValue = BalValue*100) %>%
        transform(BalValue = as.integer(BalValue)) %>%
        mutate(FairIx = fct_relevel(fct_relabel(FairIx, ~ 
        if_else(.=="Ind","Independence", paste0("Separation (",.,")"))), "Independence","Separation (TPR)")) %>%
        #if_else(.=="OAE","OAE", paste0("Sufficiency (",.,")"))), "Sufficiency (PPV)","Sufficiency (NPV)")) %>%
        mutate(Balance.Class = fct_rev( cut(BalValue,c(0,33.3,66.7,100),
                                 labels=c("Imbalanced","Undecided","Balanced"),
                                 include.lowest = TRUE, ordered_result = TRUE))) %>%
        mutate(Risk33 = if_else(BalValue < 33,"Higher risk","Lower risk"),
               Risk66 = if_else(BalValue < 66,"Higher risk","Lower risk"),
               Risk45 = if_else(BalValue < 45,"Higher risk","Lower risk"))

model_vect = rep(model,nrow(d))
d$model=model_vect
return(d)
}

lr <- model_df('lr')
svm <- model_df('svm')
knn <- model_df('knn')
rf <- model_df('rf')
d <- bind_rows(lr,svm,knn,rf)

# Box-Plot
ggplot(d,aes(x=model,y=FairValue,fill=Risk45,color=Risk45))+
  geom_boxplot(alpha=0.5,width=0.6)+
  scale_fill_manual(values = c("Firebrick","gold"),guide=FALSE)+
  scale_color_manual(values = c("Firebrick","gold"),guide=FALSE)+
  ylab("Unfair")+
  xlab("Model")+
  facet_grid(BalIx~FairIx)+
  coord_cartesian(clip = "off")+
  annotate("text", x=0.5,y=0,label=TeX("fair"), 
           parse=TRUE, size=2.5, color="gray40", hjust=0)+
  annotate("text", x=1.15,y=56,label=TeX("unfair"), 
           parse=TRUE, size=2.5, color="gray40",hjust=1)+
#  annotate("text", x=c(0,50),y=c(0,0),label=c("<-- fair","unfair -->"))+
  theme_light()+
  theme(strip.text.y = element_text(angle=0),
        axis.title.y = element_text(angle=0,hjust=0,vjust=1.05,))


#ggsave('images/run1 unfairness distrib. risk 33 models, baseline, base paper.png', plot = last_plot())
```



```{r models Box-plot} 
# base paper analysis, differences between baseline and smote 
run = 'run1'
base_paper = 'base paper measures/'  # base paper measures/  OR 'all attributes' (for no base paper)
#model = 'lr'

# Compas and Juvenile removal
model_df <- function(model, smote=FALSE) {
imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) %>%
                        slice(9:29) %>%  #not compas and juvenile
                        select("Dataset -", "Attribute", "Gini","Shannon","Simpson", "IIR")

# "Diff Ind" ,"Diff TPR","Diff FPR"
# "Diff PPV","Diff NPV", "Diff OAE"
fairIx = c("Diff Ind" ,"Diff TPR","Diff FPR", "Diff PPV","Diff NPV", "Diff OAE")       
# or 'measures smote' # ATTENTION HERE

if(smote){
  unfairness_measures <- readODS::read_ods(paste(base_paper,"measures smote/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(9:29) %>%
                        select("Dataset -", "Attribute", "m",fairIx)  
}else{
  unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(9:29) %>%
                        select("Dataset -", "Attribute", "m",fairIx)
}
        
d <- merge(imbalance_measures, unfairness_measures) %>%
        rename(Dataset=1) %>%
        mutate(Dataset=str_replace(Dataset," *- *","")) %>%
        pivot_longer(starts_with("Diff"),names_to="FairIx", 
                   names_prefix="Diff ",values_to="FairValue")%>%
        transform(FairValue = FairValue*100) %>%
        transform(FairValue = as.integer(FairValue)) %>%
        pivot_longer(c("Gini","Shannon","Simpson","IIR"),names_to="BalIx", 
                   values_to="BalValue") %>%
        transform(BalValue = BalValue*100) %>%
        transform(BalValue = as.integer(BalValue)) %>%
        #mutate(FairIx = fct_relevel(fct_relabel(FairIx, ~ 
        #if_else(.=="Ind","Independence", paste0("Separation (",.,")"))), "Independence","Separation (TPR)")) %>%
        #if_else(.=="OAE","OAE", paste0("Sufficiency (",.,")"))), "Sufficiency (PPV)","Sufficiency (NPV)")) %>%
        mutate(Balance.Class = fct_rev( cut(BalValue,c(0,33.3,66.7,100),
                                 labels=c("Imbalanced","Undecided","Balanced"),
                                 include.lowest = TRUE, ordered_result = TRUE))) %>%
        mutate(Risk33 = if_else(BalValue < 33,"Higher risk","Lower risk"),
               Risk66 = if_else(BalValue < 66,"Higher risk","Lower risk"))

model_vect = rep(model,nrow(d))
d$model=model_vect

if(smote){
  model_type = rep('smote',nrow(d))
  d$type=model_type
}else{
  model_type = rep('baseline',nrow(d))
  d$type=model_type
}
return(d)
}

#svm <- model_df('svm', smote=TRUE)
#knn <- model_df('knn', smote=TRUE)
#rf <- model_df('rf', smote=TRUE)

baseline <- model_df('rf')
smote <- model_df('rf', smote=TRUE)
d <- bind_rows(baseline,smote)

d$FairIx_f = factor(d$FairIx, levels=c('Ind','TPR','FPR','PPV','NPV','OAE'))

# Box-Plot
ggplot(d,aes(x=type,y=FairValue,fill=Risk33,color=Risk33))+
  geom_boxplot(alpha=0.5,width=0.6)+
  scale_fill_manual(values = c("Firebrick","gold"),guide=FALSE)+
  scale_color_manual(values = c("Firebrick","gold"),guide=FALSE)+
  ylab("Unfair")+
  xlab("Type")+
  facet_grid(FairIx_f~ BalIx)+  # labeller = labeller(FairIx = fairIx_names)
  coord_cartesian(clip = "off")+
  annotate("text", x=0.5,y=0,label=TeX("fair"), 
           parse=TRUE, size=2.5, color="gray40", hjust=0)+
  annotate("text", x=1.15,y=56,label=TeX("unfair"), 
           parse=TRUE, size=2.5, color="gray40",hjust=1)+
#  annotate("text", x=c(0,50),y=c(0,0),label=c("<-- fair","unfair -->"))+
  theme_light()+
  theme(strip.text.y = element_text(angle=0),
        axis.title.y = element_text(angle=0,hjust=0,vjust=1.05,))


ggsave('images/boxplot differences smote rf.png', plot = last_plot())
```




```{r models Box-plot} 
# base paper analysis, differences among different thresholds (Box plot distributions)
run = 'run1'
base_paper = 'base paper measures/'  # base paper measures/  OR 'all attributes' (for no base paper)

# Compas and Juvenile removal
model_df <- function(model, smote=FALSE) {
imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) %>%
                        slice(9:29) %>%  #not compas and juvenile
                        select("Dataset -", "Attribute", "Gini","Shannon","Simpson", "IIR")

# "Diff Ind" ,"Diff TPR","Diff FPR"
# "Diff PPV","Diff NPV", "Diff OAE"
fairIx = c("Diff Ind" ,"Diff TPR","Diff FPR", "Diff PPV","Diff NPV", "Diff OAE")       
# or 'measures smote' # ATTENTION HERE

if(smote){
  unfairness_measures <- readODS::read_ods(paste(base_paper,"measures smote/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(9:29) %>%
                        select("Dataset -", "Attribute", "m",fairIx)  
}else{
  unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(9:29) %>%
                        select("Dataset -", "Attribute", "m",fairIx)
}
        
d <- merge(imbalance_measures, unfairness_measures) %>%
        rename(Dataset=1) %>%
        mutate(Dataset=str_replace(Dataset," *- *","")) %>%
        pivot_longer(starts_with("Diff"),names_to="FairIx", 
                   names_prefix="Diff ",values_to="FairValue")%>%
        transform(FairValue = FairValue*100) %>%
        transform(FairValue = as.integer(FairValue)) %>%
        pivot_longer(c("Gini","Shannon","Simpson","IIR"),names_to="BalIx", 
                   values_to="BalValue") %>%
        transform(BalValue = BalValue*100) %>%
        transform(BalValue = as.integer(BalValue)) %>%
        #mutate(FairIx = fct_relevel(fct_relabel(FairIx, ~ 
        #if_else(.=="Ind","Independence", paste0("Separation (",.,")"))), "Independence","Separation (TPR)")) %>%
        #if_else(.=="OAE","OAE", paste0("Sufficiency (",.,")"))), "Sufficiency (PPV)","Sufficiency (NPV)")) %>%
        mutate(Balance.Class = fct_rev( cut(BalValue,c(0,33.3,66.7,100),
                                 labels=c("Imbalanced","Undecided","Balanced"),
                                 include.lowest = TRUE, ordered_result = TRUE)))


model_vect = rep(model,nrow(d))
d$model=model_vect

if(smote){
  model_type = rep('smote',nrow(d))
  d$type=model_type
}else{
  model_type = rep('baseline',nrow(d))
  d$type=model_type
}

return(d)
}

risk_df <- function(model, risk=33) {
  d <- model_df(model)
  d <- d %>% mutate(Risk = if_else(BalValue < risk,"Higher risk","Lower risk"))
  print(as.character(risk))
  risk_type_vect = rep(as.character(risk), nrow(d))
  d$risk_type=risk_type_vect
  return(d)
}

model = 'rf'
d <- bind_rows(risk_df(model, 25), risk_df(model, 33), risk_df(model, 50),
               risk_df(model, 75)) #, risk_df(model, 80)) 

d$FairIx_f = factor(d$FairIx, levels=c('Ind','TPR','FPR','PPV','NPV','OAE'))

# Box-Plot
ggplot(d,aes(x=risk_type,y=FairValue,fill=Risk,color=Risk))+
  geom_boxplot(alpha=0.5,width=0.6)+
  scale_fill_manual(values = c("Firebrick","gold"),guide=FALSE)+
  scale_color_manual(values = c("Firebrick","gold"),guide=FALSE)+
  ylab("Unfair")+
  xlab("Type")+
  facet_grid(FairIx_f~BalIx)+
  coord_cartesian(clip = "off")+
  annotate("text", x=0.5,y=0,label=TeX("fair"), 
           parse=TRUE, size=2.5, color="gray40", hjust=0)+
  annotate("text", x=1.15,y=56,label=TeX("unfair"), 
           parse=TRUE, size=2.5, color="gray40",hjust=1)+
#  annotate("text", x=c(0,50),y=c(0,0),label=c("<-- fair","unfair -->"))+
  theme_light()+
  theme(strip.text.y = element_text(angle=0),
        axis.title.y = element_text(angle=0,hjust=0,vjust=1.05,))


ggsave('images/thresholds comparison rf.png', plot = last_plot())
```


-------------------------------------------------------

```{r models Box-plot} 
# without compas, juvenile, community crimes and age category extensive analysis
run = 'run1'
base_paper = 'all attributes measures/'  # base paper measures/  OR 'all attributes' (for no base paper)
#model = 'lr'

# Compas and Juvenile removal
model_df <- function(model) {
imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) %>%
                        slice(10:38) %>% # 29 or 38 (38 -> without Compas, juvenile, community and Age category)
                        filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>%
                        select("Dataset -", "Attribute", "Gini","Shannon","Simpson", "IIR")
                        
fairIx = c("Diff Ind" ,"Diff TPR","Diff FPR")       
# or 'measures smote'                                      # ATTENTION HERE
unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(10:38) %>% # 29 or 38 (38 -> without Compas, juvenile, community and Age category)
                        filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>%
                        select("Dataset -", "Attribute", "m",fairIx)  
# "Diff Ind" ,"Diff TPR","Diff FPR"
# "Diff PPV","Diff NPV", "Diff OAE"
        
d <- merge(imbalance_measures, unfairness_measures) %>%
        rename(Dataset=1) %>%
        mutate(Dataset=str_replace(Dataset," *- *","")) %>%
        pivot_longer(starts_with("Diff"),names_to="FairIx", 
                   names_prefix="Diff ",values_to="FairValue")%>%
        transform(FairValue = FairValue*100) %>%
        transform(FairValue = as.integer(FairValue)) %>%
        pivot_longer(c("Gini","Shannon","Simpson","IIR"),names_to="BalIx", 
                   values_to="BalValue") %>%
        transform(BalValue = BalValue*100) %>%
        transform(BalValue = as.integer(BalValue)) %>%
        mutate(FairIx = fct_relevel(fct_relabel(FairIx, ~ 
        if_else(.=="Ind","Independence", paste0("Separation (",.,")"))), "Independence","Separation (TPR)")) %>%
        #if_else(.=="OAE","OAE", paste0("Sufficiency (",.,")"))), "Sufficiency (PPV)","Sufficiency (NPV)")) %>%
        mutate(Balance.Class = fct_rev( cut(BalValue,c(0,33.3,66.7,100),
                                 labels=c("Imbalanced","Undecided","Balanced"),
                                 include.lowest = TRUE, ordered_result = TRUE))) %>%
        mutate(Risk33 = if_else(BalValue < 33,"Higher risk","Lower risk"),
               Risk66 = if_else(BalValue < 66,"Higher risk","Lower risk"),
               Risk45 = if_else(BalValue < 45,"Higher risk","Lower risk"))

model_vect = rep(model,nrow(d))
d$model=model_vect
return(d)
}

lr <- model_df('lr')
svm <- model_df('svm')
knn <- model_df('knn')
rf <- model_df('rf')
d <- bind_rows(lr,svm,knn,rf)

# Box-Plot
ggplot(d,aes(x=model,y=FairValue,fill=Risk45,color=Risk45))+
  geom_boxplot(alpha=0.5,width=0.6)+
  scale_fill_manual(values = c("Firebrick","gold"),guide=FALSE)+
  scale_color_manual(values = c("Firebrick","gold"),guide=FALSE)+
  ylab("Unfair")+
  xlab("Model")+
  facet_grid(BalIx~FairIx)+
  coord_cartesian(clip = "off")+
  annotate("text", x=0.5,y=0,label=TeX("fair"), 
           parse=TRUE, size=2.5, color="gray40", hjust=0)+
  annotate("text", x=1.15,y=56,label=TeX("unfair"), 
           parse=TRUE, size=2.5, color="gray40",hjust=1)+
#  annotate("text", x=c(0,50),y=c(0,0),label=c("<-- fair","unfair -->"))+
  theme_light()+
  theme(strip.text.y = element_text(angle=0),
        axis.title.y = element_text(angle=0,hjust=0,vjust=1.05,))


ggsave('images/run1 unfairness distrib. risk 33 models, baseline, extensive Ind,TPR,FPR.png', plot = last_plot())
```

```{r models Box-plot} 
# extensive analysis, differences among different thresholds (Box plot distributions)
run = 'run1'
base_paper = 'all attributes measures/'  # base paper measures/  OR 'all attributes' (for no base paper)

# Compas and Juvenile removal
model_df <- function(model, smote=FALSE) {
imbalance_measures <- readODS::read_ods(paste(base_paper,"balance measures.ods", sep=''), run) %>%
                        slice(10:38) %>% # 29 or 38 (38 -> without Compas, juvenile, community and Age category)
                        filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>%
                        select("Dataset -", "Attribute", "Gini","Shannon","Simpson", "IIR")

# "Diff Ind" ,"Diff TPR","Diff FPR"
# "Diff PPV","Diff NPV", "Diff OAE"
fairIx = c("Diff Ind" ,"Diff TPR","Diff FPR", "Diff PPV","Diff NPV", "Diff OAE")       
# or 'measures smote' # ATTENTION HERE

if(smote){
  unfairness_measures <- readODS::read_ods(paste(base_paper,"measures smote/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(10:38) %>% # 29 or 38 (38 -> without Compas, juvenile, community and Age category)
                        filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>%
                        select("Dataset -", "Attribute", "m",fairIx)  
}else{
  unfairness_measures <- readODS::read_ods(paste(base_paper,"measures/unfairness measures ", model, ".ods", sep=''),run) %>%
                        slice(10:38) %>% # 29 or 38 (38 -> without Compas, juvenile, community and Age category)
                        filter(Attribute != 'Age category' & Attribute != 'Age cateogry') %>%
                        select("Dataset -", "Attribute", "m",fairIx)
}
        
d <- merge(imbalance_measures, unfairness_measures) %>%
        rename(Dataset=1) %>%
        mutate(Dataset=str_replace(Dataset," *- *","")) %>%
        pivot_longer(starts_with("Diff"),names_to="FairIx", 
                   names_prefix="Diff ",values_to="FairValue")%>%
        transform(FairValue = FairValue*100) %>%
        transform(FairValue = as.integer(FairValue)) %>%
        pivot_longer(c("Gini","Shannon","Simpson","IIR"),names_to="BalIx", 
                   values_to="BalValue") %>%
        transform(BalValue = BalValue*100) %>%
        transform(BalValue = as.integer(BalValue)) %>%
        #mutate(FairIx = fct_relevel(fct_relabel(FairIx, ~ 
        #if_else(.=="Ind","Independence", paste0("Separation (",.,")"))), "Independence","Separation (TPR)")) %>%
        #if_else(.=="OAE","OAE", paste0("Sufficiency (",.,")"))), "Sufficiency (PPV)","Sufficiency (NPV)")) %>%
        mutate(Balance.Class = fct_rev( cut(BalValue,c(0,33.3,66.7,100),
                                 labels=c("Imbalanced","Undecided","Balanced"),
                                 include.lowest = TRUE, ordered_result = TRUE)))


model_vect = rep(model,nrow(d))
d$model=model_vect

if(smote){
  model_type = rep('smote',nrow(d))
  d$type=model_type
}else{
  model_type = rep('baseline',nrow(d))
  d$type=model_type
}

return(d)
}

risk_df <- function(model, risk=33) {
  d <- model_df(model)
  d <- d %>% mutate(Risk = if_else(BalValue < risk,"Higher risk","Lower risk"))
  print(as.character(risk))
  risk_type_vect = rep(as.character(risk), nrow(d))
  d$risk_type=risk_type_vect
  return(d)
}

model = 'rf'
d <- bind_rows(risk_df(model, 25), risk_df(model, 33), risk_df(model, 50),
               risk_df(model, 75)) #, risk_df(model, 80)) 

d$FairIx_f = factor(d$FairIx, levels=c('Ind','TPR','FPR','PPV','NPV','OAE'))

# Box-Plot
ggplot(d,aes(x=risk_type,y=FairValue,fill=Risk,color=Risk))+
  geom_boxplot(alpha=0.5,width=0.6)+
  scale_fill_manual(values = c("Firebrick","gold"),guide=FALSE)+
  scale_color_manual(values = c("Firebrick","gold"),guide=FALSE)+
  ylab("Unfair")+
  xlab("Type")+
  facet_grid(FairIx_f~BalIx)+
  coord_cartesian(clip = "off")+
  annotate("text", x=0.5,y=0,label=TeX("fair"), 
           parse=TRUE, size=2.5, color="gray40", hjust=0)+
  annotate("text", x=1.15,y=56,label=TeX("unfair"), 
           parse=TRUE, size=2.5, color="gray40",hjust=1)+
#  annotate("text", x=c(0,50),y=c(0,0),label=c("<-- fair","unfair -->"))+
  theme_light()+
  theme(strip.text.y = element_text(angle=0),
        axis.title.y = element_text(angle=0,hjust=0,vjust=1.05,))


ggsave('images/extensive thresholds comparison rf.png', plot = last_plot())
```