src/dspg17/sphadke/04_ohloh_projects_analysis.R

####################################
#### OpenHub projects: analysis ####
####################################
# This code runs profiling code and produces descriptives for project tables
# Assign the file of interest to "data" to have the code run without changing names

#### Created by: sphadke
#### Creted on: 07/06/2017
#### Last edited on: 07/07/2017


####################
#### Cleanup
####################
rm(list=ls())
gc()
set.seed(312)


####################
#### R Setup
####################
library(ggplot2)
library(gtools)
library(lubridate)
library(network)
library(reshape2)
library(scales)
library(SnowballC)
library(stringr)
library(tm)
library(wordcloud)


## Data import and rename
# random projects
load("~/git/oss/data/oss/working/openhub/randomProjects/all_random_projects_table.RData")

# relevant projects
load("~/git/oss/data/oss/working/openhub/relevantProjects/projectRelevantMaster.RData")

ran_data <- randomProjectTable
rel_data <- projectRelevantMaster
rm(randomProjectTable)
rm(projectRelevantMaster)


####################
#### Data quality check
#### And cleanup
####################

## Convert blanks and "NA"s to NAs
ran_data[ran_data == ""] <- NA
ran_data[ran_data == "\n  "] <- NA
ran_data[ran_data == "NA"] <- NA

rel_data[rel_data == ""] <- NA
rel_data[rel_data == "\n  "] <- NA
rel_data[rel_data == "NA"] <- NA


## Make relevant columns numeric
ran_data$user_count <- as.numeric(ran_data$user_count)
ran_data$average_rating <- as.numeric(ran_data$average_rating)
ran_data$rating_count <- as.numeric(ran_data$rating_count)
ran_data$review_count <- as.numeric(ran_data$review_count)
ran_data$twelve_month_contributor_count <- as.numeric(ran_data$twelve_month_contributor_count)
ran_data$total_contributor_count <- as.numeric(ran_data$total_contributor_count)
ran_data$twelve_month_commit_count <- as.numeric(ran_data$twelve_month_commit_count)
ran_data$total_commit_count <- as.numeric(ran_data$total_commit_count)
ran_data$total_code_lines <- as.numeric(ran_data$total_code_lines)


rel_data$user_count <- as.numeric(rel_data$user_count)
rel_data$average_rating <- as.numeric(rel_data$average_rating)
rel_data$rating_count <- as.numeric(rel_data$rating_count)
rel_data$review_count <- as.numeric(rel_data$review_count)
rel_data$twelve_month_contributor_count <- as.numeric(rel_data$twelve_month_contributor_count)
rel_data$total_contributor_count <- as.numeric(rel_data$total_contributor_count)
rel_data$twelve_month_commit_count <- as.numeric(rel_data$twelve_month_commit_count)
rel_data$total_commit_count <- as.numeric(rel_data$total_commit_count)
rel_data$total_code_lines <- as.numeric(rel_data$total_code_lines)

####
## Remove the fully empty rows
####
clean_ran_data <- ran_data[!apply(ran_data[2:33], 1, function(x){all(is.na(x))}), ]
rm(ran_data)

clean_rel_data <- rel_data[!apply(rel_data[2:33], 1, function(x){all(is.na(x))}), ]
rm(rel_data)

# ## Number of completely empty rows
# num_empty_rows <- length(data$licenses[apply(data[,2:ncol(data)], 1, function(x){all(is.na(x))}) == TRUE])
# rm(num_empty_rows)


## Check data quality
completeness <- matrix(NA, ncol(clean_data), 4)

for(i in 1:ncol(clean_data)){
  x <- clean_data[,i]

  completeness[i,1] <- colnames(clean_data)[i]
  completeness[i,2] <- class(x)[1]

  # Proportion of missing values
  completeness[i,3] <- round(sum(is.na(x))*100/nrow(clean_data), digits = 2)

  # How many unique values to the variable?
  completeness[i,4] <- length(unique(x))
  # summary <- summary(x)
  # if(vals <= 10){
  #   tab <- table(x)
  #   print(tab)
  # }
  #print(c(name, class, miss, vals))
}

# View(completeness)


####
#### Plot of languages
####
# ran_language_freq <- as.data.frame(table(clean_ran_data$main_language))
# ran_lang_to_plot <- head(ran_language_freq[order(ran_language_freq$Freq, decreasing= T),], n = 10)
# ran_lang_to_plot <- clean_ran_data[(clean_ran_data$main_language %in% ran_lang_to_plot$Var1) == TRUE, 'main_language']
# ran_lang_to_plot <- as.data.frame(ran_lang_to_plot)
#
#
# random_lang <- ggplot(data = ran_lang_to_plot, aes(x = ran_lang_to_plot, fill = ran_lang_to_plot)) +
#   geom_bar() +
#   theme_minimal() +
#   theme(legend.position = "none") +
#   ggtitle("Ten main languages: Random projects") +
#   labs(y = "Frequency", x = "") +
#   theme(plot.title = element_text(hjust = 0.5)) +
#   theme(text=element_text(size=20),
#         axis.text=element_text(size=16),
#         axis.title=element_text(size=20))
#
# png(filename = "./output/openhub/graphics/poster/random_main_lang.png",
#     units = "in", width = 8, height = 15,
#     res = 72, bg = "transparent")
# random_lang
# dev.off()


## Barplot of main languages
rel_language_freq <- as.data.frame(table(clean_rel_data$main_language))
rel_lang_to_plot <- head(rel_language_freq[order(rel_language_freq$Freq, decreasing= T),], n = 10)
rel_lang_to_plot <- clean_rel_data[(clean_rel_data$main_language %in% rel_lang_to_plot$Var1) == TRUE, 'main_language']
rel_lang_to_plot <- as.data.frame(rel_lang_to_plot)

rel_lang <- ggplot(data = rel_lang_to_plot, aes(x = rel_lang_to_plot, fill = rel_lang_to_plot)) +
  geom_bar() +
  scale_fill_brewer(palette = "Spectral") +
  theme_minimal() +
  theme(legend.position = "none") +
  ggtitle("Ten main languages: Relevant projects") +
  labs(y = "Frequency", x = "") +
  theme(axis.text.x = element_text(size = 24, angle = 45, hjust = 1),
        axis.text.y = element_text(size = 24),
        title = element_text(size = 24),
        axis.title = element_text(size = 20)) +
  theme(plot.title = element_text(hjust = 0.5))

png(filename = "./output/openhub/graphics/poster/rel_main_lang.png",
    units = "in", width = 8, height = 15,
    res = 72, bg = "transparent")
rel_lang
dev.off()




# Separating out the factoid columns
clean_data$age <- NA
clean_data$team_size <- NA
clean_data$activity <- NA
clean_data$comments <- NA

for(i in 1:nrow(clean_data)){

  if(length(grep("Age", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE)) > 0){
    clean_data$age[i] <- unlist(str_split(grep("Age", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE), "Age"))[2]
  }

  if(length(grep("Team", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE)) > 0){
    clean_data$team_size[i] <- unlist(str_split(grep("Team", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE), "Size"))[2]
  }

  if(length(grep("Activity", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE)) > 0){
    clean_data$activity[i] <- unlist(str_split(grep("Activity", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE), "Activity"))[2]
  }

  if(length(grep("Comments", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE)) > 0){
    clean_data$comments[i] <- unlist(str_split(grep("Comments", unlist(str_split(clean_data$factoids[i], ";")), value = TRUE), "Comments"))[2]
  }

  print(i)
}

table(clean_data$age)
table(clean_data$team_size)
table(clean_data$activity)
table(clean_data$comments)

clean_data$age <- as.factor(clean_data$age)
clean_data$activity <- as.factor(clean_data$activity)
clean_data$team_size <- as.factor(clean_data$team_size)
clean_data$comments <- as.factor(clean_data$comments)


####
#### See if things are correlated
####

# cor(clean_data[, c('average_rating', 'user_count', 'rating_count', 'review_count', 'twelve_month_contributor_count', 'total_contributor_count', 'total_commit_count', 'total_code_lines', 'twelve_month_commit_count')])


####
#### Time plot of development
####

# ggplot(clean_data, aes(created_at, total_commit_count)) +
#   geom_point() +
#   scale_x_datetime(date_labels = "%Y-%b") #+ xlab("") + ylab("Daily Views")
#
# ggplot(clean_data, aes(created_at, total_contributor_count)) +
#   geom_line() +
#   scale_x_datetime(date_labels = "%Y-%b")




####
#### Wordclouds
####
## Main development language

language_corpus <- Corpus(VectorSource(clean_data$main_language))
language_dtm <- TermDocumentMatrix(language_corpus)
dps_m <- as.matrix(language_dtm)
dps_m <- sort(rowSums(dps_m),decreasing=TRUE)
dps_m <- data.frame(word = names(dps_m),freq = dps_m)

# png(filename="/wordcloud_dps.png",
#     units="in",
#     width=10,
#     height=10,
#     #pointsize=12,
#     res=72
# )

ggplot(data = clean_data, aes(x = main_language)) +
  geom_bar()

wordcloud(dps_m$word, dps_m$freq, min.freq = 1, random.order = FALSE, colors=brewer.pal(8, "Dark2"))
# dev.off()


## Top 3 languages
all_languages <- c(unlist(str_split(clean_data$languages, ";")))
all_languages <- all_languages[str_detect(all_languages, "other") == FALSE]
language_corpus <- Corpus(VectorSource(all_languages))
language_corpus <- tm_map(language_corpus, removeNumbers)
language_corpus <- tm_map(language_corpus, removeWords, c("other", "Other"))
language_dtm <- TermDocumentMatrix(language_corpus)
dps_m <- as.matrix(language_dtm)
dps_m <- sort(rowSums(dps_m),decreasing=TRUE)
dps_m <- data.frame(word = names(dps_m),freq = dps_m)

wordcloud(dps_m$word, dps_m$freq, min.freq = 1, random.order = FALSE, colors=brewer.pal(8, "Dark2"))


####
## Later
####
## Description
dps_chrg_corpus <- Corpus(VectorSource(clean_data$description))
dps_chrg_corpus <- tm_map(dps_chrg_corpus, PlainTextDocument)
dps_chrg_corpus <- tm_map(dps_chrg_corpus, content_transformer(tolower))

dps_chrg_corpus <- tm_map(dps_chrg_corpus, removeWords, stopwords('english'))
dps_chrg_corpus <- tm_map(dps_chrg_corpus, removeWords, c("will","set","functions", "data", "package", "based", "can", "provides", "set", "used", "project", "using", "contains", "function"))
dps_chrg_corpus <- tm_map(dps_chrg_corpus, removeNumbers)
dps_chrg_corpus <- tm_map(dps_chrg_corpus, removePunctuation)

dps_chrg_corpus <- tm_map(dps_chrg_corpus, stemDocument)

dps_temp_stemmed <- data.frame(text = sapply(dps_chrg_corpus, as.character), stringsAsFactors = FALSE)

oh_random_proj_desc_dtm <- TermDocumentMatrix(dps_chrg_corpus)
save(oh_random_proj_desc_dtm, file = "~/git/oss/data/oss/working/openhub/randomProjects/oh_random_proj_desc_dtm.RData")
dps_m <- as.matrix(dps_dtm)
dps_m <- sort(rowSums(dps_m),decreasing=TRUE)
dps_m <- data.frame(word = names(dps_m),freq = dps_m)

wordcloud(dps_m$word, dps_m$freq, min.freq = 20, random.order = FALSE, colors=brewer.pal(8, "Dark2"))



####
#### Network of tags
####

tags_df <- na.omit(clean_data$tags)

tag_edgelist <- matrix(ncol = 2)

for (i in 1:nrow(clean_data)){
  num_tags <- length(unlist(str_split(tags_df[i], pattern = ";")))

  if(num_tags > 1) {
    val_tags <- unlist(str_split(tags_df[i], pattern = ";"))

    combinations <- combinations(n = num_tags, r = 2, val_tags, repeats.allowed = FALSE, set = TRUE)

    tag_edgelist <- rbind(tag_edgelist, combinations)
  }

  print(i)
}

tag_edgelist <- na.omit(tag_edgelist)
tag_net <- network(as.data.frame(tag_edgelist), directed = FALSE, matrix.type = "edgelist")

plot(tag_net, displaylabels = F,
     #label = get.vertex.attribute(contact.net, "female"),
     vertex.cex = 1)#, vertex.col = c("Blue", "Red"))
uva-bi-sdad/oss documentation built on June 5, 2019, 12:37 a.m.