In benjaminrobinson/LearnDC: LearnDC

This is the script that Ben wrote to clean up the released PARCC data. The output of this script are saved as data objects inside this package - school_parcc, lea_parcc, etc.

library(dplyr)
library(tidyr)
library(jsonlite)
options(stringsAsFactors=FALSE)
options(scipen=20)
`%notin%` <- function(x,y) !(x %in% y) 
leadgr <- function(x, y){
  if(!is.na(x)){
    while(nchar(x)<y){
      x <- paste("0",x,sep="")
    }
  }
  return(x)
}

parcc <- rbind(read.csv('https://s3.amazonaws.com/tembo-osse-web/osse-nga-data.csv'),read.csv("https://s3.amazonaws.com/tembo-osse-web/osse-nga-data-2015-16.csv",skipNul = T))

names(parcc) <- tolower(names(parcc))
names(parcc)[1:2] <- c('lea_code','school_code')
names(parcc)[7] <- c('grade')
names(parcc)[9:10] <- c('subgroup_type','subgroup')

subgroup_map <- c("BL7"="Black/African American",
                  "WH7"="White",
                  "HI7"="Hispanic",
                  "AS7"="Asian",
                  "MU7"="Multiracial",
                  "PI7"="Pacific Islander",
                  "AM7"="American Indian",
                  "All"="All",
                  "Economically Disadvantaged"="Economically Disadvantaged",
                  "Limited English Proficient"="Limited English Proficient",
                  "Female"="Female",
                  "Male"="Male",
                  "Not Economically Disadvantaged"="Not Economically Disadvantaged",
                  "Not Limited English Proficient"="Not Limited English Proficient",
                  "Not Special Education"="Not Special Education",
                  "Special Education"="Special Education",
                  "Subclaim 1"="Subclaim 1",
                  "Subclaim 2"="Subclaim 2",
                  "Subclaim 3"="Subclaim 3",
                  "Subclaim 4"="Subclaim 4",
                  "Subclaim 5"="Subclaim 5"
                  )
parcc$subgroup <- subgroup_map[parcc$subgroup]
parcc <- subset(parcc,!is.na(value))
parcc$metric <- tolower(parcc$metric)


grade_map <- c('All'='All',
                      '11'='11th Grade',
                      '3'='3rd Grade',
                      '3-8'='3rd-8th Grade',
                      '4'='4th Grade',
                      '5'='5th Grade',
                      '6'='6th Grade',
                      '7'='7th Grade',
                      '8'='8th Grade',
                      '9-12'='9th-12th Grade',
                      'Algebra I'='Algebra I',
                      'Algebra II'='Algebra II',
                      'English I'='English I',
                      'English II'='English II',
                      'Integrated Math II'='Integrated Math II',
                      'Geometry'='Geometry',
                      'All Grades'='All',
                      'Grade 11'='11th Grade',
                      'Grade 3'='3rd Grade',
                      'Grade 4'='4th Grade',
                      'Grade 5'='5th Grade',
                      'Grade 6'='6th Grade',
                      'Grade 7'='7th Grade',
                      'Grade 8'='8th Grade',
                      'Grades 3-8'='3rd Grade - 8th Grade',
                      'Grades 9-12'='9th Grade - 12th Grade'
                      )
parcc$grade <- grade_map[parcc$grade]

parcc <- parcc %>%
mutate(lea_code=sapply(lea_code,leadgr,4),
lea_code=ifelse(lea_code %in% "0All","All",lea_code),
school_code=sapply(school_code,leadgr,4),
school_code=ifelse(school_code %in% '0All','All',school_code)) 

parcc <- parcc %>%
left_join(subset(fromJSON("https://learndc-api.herokuapp.com//api/schools?sha=promoted")[2:3],org_code %in% parcc$school_code) %>%
rename(school_code=org_code,school_name=org_name)) %>%
    mutate(school_name=ifelse(school_code=='0219','Bunker Hill ES',
                              ifelse(school_code=='0304','River Terrace ES',
                                     ifelse(school_code=='0347','Brookland MS',
                                            ifelse(school_code=='0349','Dorothy Height ES',school_name)))))
parcc <- parcc %>%
left_join(subset(fromJSON("https://learndc-api.herokuapp.com//api/leas?sha=promoted")[2:3],org_code %in% parcc$lea_code) %>%
rename(lea_code=org_code,lea_name=org_name)) %>% select(1,14,2,13,3:12) %>% mutate(school_name=ifelse(is.na(school_name),'All',school_name),
lea_name=ifelse(lea_code=='All','All',lea_name))

parcc$subject <- ifelse(parcc$subject %in% 'ELA','Reading',parcc$subject)
parcc$year <- paste0(parcc$year-1,"-",parcc$year)
parcc$subgroup_type <- NULL

parcc <- parcc %>% distinct %>% spread(metric,value)
parcc[,12:18] <- parcc[,12:18]/100

names(parcc)[12:18] <- gsub(" ","_",names(parcc)[12:18])
names(parcc)[18] <- 'percent_proficient_3+'

school <- parcc %>% filter(school_code %notin% 'All')
lea <- parcc %>% filter(lea_code %notin% c('All','0001') & school_code %in% 'All')
sector <- parcc %>% filter(lea_code %in% '0001' & school_code %in% 'All')
state <- parcc %>% filter(lea_code %in% 'All' & school_code %in% 'All')

setwd(".../LearnDC/data")
save(school,file="school_parcc.rda")
save(lea,file="lea_parcc.rda")
save(sector,file="sector_parcc.rda")
save(state,file="state_parcc.rda")

setwd(".../LearnDC/data-raw")
write.csv(school,"school_parcc.csv")
write.csv(lea,"lea_parcc.csv")
write.csv(sector,"sector_parcc.csv")
write.csv(state,"state_parcc.csv")