In okamumu/msrat: A Metrics-Based Software Reliability Assessment Tool

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.path = "man/figures/README-",
  out.width = "100%"
)

msrat

msrat is a tool designed to assess software reliability based on fault data gathered during the testing phase. This tool utilizes two types of metrics: d-metrics and s-metrics, both essential for a comprehensive evaluation.

The d-metrics include variables like the number of test cases and coverage levels, recorded over specific time periods (e.g., daily or weekly). These metrics primarily reflect the software testing environment and are subject to change as testing progresses, capturing the dynamic nature of the testing process.

On the other hand, s-metrics focus on the software product itself. These include metrics such as the number of code lines and the complexity of the software. Unlike d-metrics, s-metrics are generally stable throughout the testing process, providing a consistent basis for evaluating the software's inherent qualities.

Installation

You can install msrat from GitHub with:

install.packages("devtools")
devtools::install_github("SwReliab/msrat")

Alternatively, you can install msrat with remotes:

install.packages("remotes")
remotes::install_github("SwReliab/msrat")

Example

This is an example for reliability evaluation with d-metrics.

### load library
library(msrat)

### load example data for d-metrics
data(dmet)

### dmet.ds1 consists of the number of days (day), the number of faults (fault), the number of testcases (tc), the cumulative number of testcases (ctc), the insrease of C0 coverage (cov) and C0 coverage (ccov).
dmet.ds1

### Esimate the model for d-metrics
(result <- fit.srm.logit(formula=fault~., data=dmet.ds1))

### Select metrics in terms of AIC
(result.aic <- step(result))

### Draw the number of faults for each day and the number of faults estimated by the model (dmvplot is in Rsrat package)
dmvfplot(fault=dmet.ds1$fault, srms=result.aic)

This is an example for reliability evaluation with s-metrics.

### load example data for s-metrics
data(tomcat5)

### S-metrics for modules: catalina, connector, jasper, servlets, tester and webapps
#### LOC: Lines of code
#### St: The number of statements
#### Br: The number of branches
#### Co: Percentage of comments
#### Fn: The number of functions (methods)
#### Mc: Maximum complexity
#### Ac: Average complexity
tomcat5.smet

### Each module has the fault data (grouped data)
tomcat5.catalina

### List for fault data
data.tomcat5 <- list(catalina=tomcat5.catalina,
                     connector=tomcat5.connector,
                     jasper=tomcat5.jasper,
                     servlets=tomcat5.servlets,
                     tester=tomcat5.tester,
                     webapps=tomcat5.webapps)

### Estimate NHPP model (gamma model) parameters using Rsrt
# library(Rsrat)
(results.tomcat5 <- lapply(data.tomcat5,
                          function(d) fit.srm.nhpp(time=d$time,
                                                   fault=d$fault,
                                                   srm.names=c("gamma"))))

### Estimate s-metrics (LOC and St) parameters
(result <- fit.srm.poireg(formula=~LOC+St, data=tomcat5.smet, srms=results.tomcat5))

### Draw 
mvfplot(time=time, fault=fault, data=tomcat5.catalina,
        srms=list(results.tomcat5$catalina, result$srm$srms$catalina))

mvfplot(time=time, fault=fault, data=tomcat5.connector,
        srms=list(results.tomcat5$connector, result$srm$srms$connector))

mvfplot(time=time, fault=fault, data=tomcat5.jasper,
        srms=list(results.tomcat5$jasper, result$srm$srms$jasper))

mvfplot(time=time, fault=fault, data=tomcat5.servlets,
        srms=list(results.tomcat5$servlets, result$srm$srms$servlets))

mvfplot(time=time, fault=fault, data=tomcat5.tester,
        srms=list(results.tomcat5$tester, result$srm$srms$tester))

mvfplot(time=time, fault=fault, data=tomcat5.webapps,
        srms=list(results.tomcat5$webapps, result$srm$srms$webapps))

Examples for penalized MLE

This is an example by applying the penalized MLE on regression coefficients.

### load library
library(msrat)

### load example data for d-metrics
data(dmet)

### dmet.ds1 consists of the number of days (day), the number of faults (fault), the number of testcases (tc), the cumulative number of testcases (ctc), the insrease of C0 coverage (cov) and C0 coverage (ccov).
dmet.ds1

### Esimate the model for d-metrics
### lambda is the hyper parameter to determine the magniture of penalty.
### alpha is the parameter to determine the lorm; alpha=1 means L1 norm, alpha=0 indicates L2 norm
(result <- fit.srm.logit.penalized(formula=fault~., data=dmet.ds1, lambda=0.01, alpha=1))

### Draw the number of faults for each day and the number of faults estimated by the model (dmvplot is in Rsrat package)
dmvfplot(fault=dmet.ds1$fault, srms=list(result$srm))

### load example data for s-metrics
data(tomcat5)

### S-metrics for modules: catalina, connector, jasper, servlets, tester and webapps
#### LOC: Lines of code
#### St: The number of statements
#### Br: The number of branches
#### Co: Percentage of comments
#### Fn: The number of functions (methods)
#### Mc: Maximum complexity
#### Ac: Average complexity
tomcat5.smet

### Each module has the fault data (grouped data)
tomcat5.catalina

### List for fault data
data.tomcat5 <- list(catalina=tomcat5.catalina,
                     connector=tomcat5.connector,
                     jasper=tomcat5.jasper,
                     servlets=tomcat5.servlets,
                     tester=tomcat5.tester,
                     webapps=tomcat5.webapps)

### Estimate NHPP model (gamma model) parameters using Rsrt
# library(Rsrat)
(results.tomcat5 <- lapply(data.tomcat5,
                          function(d) fit.srm.cph(time=d$time,
                                                   fault=d$fault,
                                                   phase=1:10)))

### Estimate s-metrics parameters with penalty
(result <- fit.srm.poireg.penalized(formula=~., data=tomcat5.smet, srms=results.tomcat5, lambda=1))

### Draw 
mvfplot(time=time, fault=fault, data=tomcat5.catalina,
        srms=list(results.tomcat5$catalina, result$srm$srms$catalina))

mvfplot(time=time, fault=fault, data=tomcat5.connector,
        srms=list(results.tomcat5$connector, result$srm$srms$connector))

mvfplot(time=time, fault=fault, data=tomcat5.jasper,
        srms=list(results.tomcat5$jasper, result$srm$srms$jasper))

mvfplot(time=time, fault=fault, data=tomcat5.servlets,
        srms=list(results.tomcat5$servlets, result$srm$srms$servlets))

mvfplot(time=time, fault=fault, data=tomcat5.tester,
        srms=list(results.tomcat5$tester, result$srm$srms$tester))

mvfplot(time=time, fault=fault, data=tomcat5.webapps,
        srms=list(results.tomcat5$webapps, result$srm$srms$webapps))