In kanishkamisra/footrulr: Measure Quality of Machine Generated Texts

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)

library(footrulr)
library(dplyr)
library(ggplot2)
library(purrr)

Read data

directory <- "https://raw.githubusercontent.com/kanishkamisra/footrulr/master/data-raw/"

systems <- c("newstest2018.LMU-nmt.5756.de-en", "newstest2018.JHU.5706.de-en", "newstest2018.LMU-unsup.5650.de-en")

references <- c("newstest2018-deen-ref-ts.en", "newstest2018-deen-ref-ts2.en")

system_results <- map_dfr(systems, function(x) {
  system <- footrulr(paste0(directory, x), paste0(directory, references))
  system <- system[1:100]
  bleu(system)
}, .id = "system")

system_results

Compute BLEU scores by system

bleu_results <- system_results %>%
  group_by(system, candidate) %>%
  summarize(bleu = mean(scores, na.rm = TRUE))

bleu_results

Plot BLEU Score distribution per system

bleu_results %>%
  ggplot(aes(bleu, fill = system)) +
  geom_histogram() +
  facet_wrap(~system, nrow = 2, ncol = 2) +
  scale_fill_viridis_d() +
  theme_light() +
  theme(
    legend.position = "top"
  )