1. Repeat the exercise from the Batch Processing Lecture (5 April), but do it using real data sets rather than purely simulated. Check with folks in your lab to see if there are multiple data sets available for analysis, or ask Nick, Lauren, or Emily for suggestions for other data sources. Stick to simple data analyses and graphics, but try to set it up as a batch process that will work on multiple files and save summary results to a common file.
# ----------------------------------------
# FUNCTION file_builder
# description: create a set of random files for regression
# inputs: file_n = number of files to create
# file_folder = name of folder for random files
# file_size = c(min,max) number of rows in file
# file_na = number on average of NA values per column
# outputs: set of random files
##########################################
file_builder = function(file_n=10,
file_folder="RandomFiles/",
file_size=c(15,100),
file_na=3) {
for (i in seq_len(file_n)) {
file_length = sample(file_size[1]:file_size[2],size=1)
var_x = runif(file_length)
var_y = runif(file_length)
df = data.frame(var_x,var_y)
bad_vals = rpois(n=1,lambda=file_na)
df[sample(nrow(df),size=bad_vals),1] = NA
df[sample(nrow(df),size=bad_vals),2] = NA
file_label = paste(file_folder,
"ranFile",
formatC(i,
width=3,
format="d",
flag="0"),
".csv",sep="")
write.table(cat("# Simulated random data file for batch processing","\n",
"# timestamp: ",as.character(Sys.time()),"\n",
"# NJG","\n",
"# ------------------------", "\n",
"\n",
file=file_label,
row.names="",
col.names="",
sep=""))
write.table(x=df,
file=file_label,
sep=",",
row.names=FALSE,
append=TRUE)
}
} # end of file_builder
# ----------------------------------------
# ----------------------------------------
# FUNCTION reg_stats
# description: fits linear model, extracts stats
# inputs: 2-column data frame (x and y)
# outputs: slope, p-value, and r2
##########################################
reg_stats = function(d=NULL) {
if(is.null(d)) {
x_var = runif(10)
y_var = runif(10)
d = data.frame(x_var,y_var)
}
. = lm(data=d,d[,2]~d[,1])
. = summary(.)
stats_list = list(slope=.$coefficients[2,1],
p_val=.$coefficients[2,4],
r2=.$r.squared)
return(stats_list)
} # end of reg_stats
# ----------------------------------------
# Global variables ---------------------------
file_folder = "RandomFiles/"
n_files = 100
file_out = "StatsSummary.csv"
#####################################
dir.create(file_folder)
## Warning in dir.create(file_folder): 'RandomFiles' already exists
file_builder(file_n=n_files)
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
## ""
## Warning in write.table(x = df, file = file_label, sep = ",", row.names =
## FALSE, : appending column names to file
file_names = list.files(path=file_folder)
ID = seq_along(file_names)
file_name = file_names
slope = rep(NA,n_files)
p_val = rep(NA,n_files)
r2 = rep(NA,n_files)
stats_out = data.frame(ID,file_name,slope,p_val,r2)
for (i in seq_along(file_names)) {
data = read.table(file=paste(file_folder,file_names[i],sep=""),
sep=",",
header=TRUE)
d_clean = data[complete.cases(data),]
. = reg_stats(d_clean)
stats_out[i,3:5] = unlist(.)
}
write.table(cat("# Summary stats for ",
"batch processing of regression models","\n",
"# timestamp: ",as.character(Sys.time()),"\n",
"# NJG","\n",
"# ------------------------", "\n",
"\n",
file=file_out,
row.names="",
col.names="",
sep=""))
## ""
write.table(x=stats_out,
file=file_out,
row.names=FALSE,
col.names=TRUE,
sep=",",
append=TRUE)
## Warning in write.table(x = stats_out, file = file_out, row.names = FALSE, :
## appending column names to file