Courtney J. Smith

Oddly Specific, Potentially Useful Code - Data Analysis R snippets

2023-08-17T00:00:00-07:00

This is a collection of oddly specific, potential useful code snippets that I use frequently enough in my general data analysis work in R that I wanted to put them in one place as a reference. Some of these things can be a bit of a pain to figure out or look up, so I wanted to share in case it helps someone else.

Each of the below code sets is completely independent from one another if under a separate comment (#), even within a given code block, except for the initial loading of the dplyr package.

Code for various specific filtering rows and subseting columns tasks

library(dplyr)

# df keep row with lowest of a certain value for each group
newdf <- as.data.table(mydf)[ , .SD[which.min(P)], by = Gene]

# If have pairwise combinations of things in two separate columns and want to filter down to only a single row for each unique pair (where order doesn't matter)
mydf <- mydf[!duplicated(t(apply(mydf %>% select(ID,Item.1,Item.2), 1, sort))),]

# subset rows in a df based on having a certain number of columns that meet a criteria
sigthres = 5e-8
ncut = 1
pvals <- mydif %>% select(matches("^P_VALUE_")) # make a new df that only has the pvalue columns
newdf <- mydif[apply(pvals, 1, function(r) sum(r < sigthres) >= ncut),] # only keep rows where the pval is < sigthres for ncut or more columns

# select top n of a group
n=2
newdf <- as.data.frame(mydf %>% group_by(Col1,Col2) %>% arrange(Col1,Col2,desc(myvalue)) %>% slice(1:n) # filter to only keep top two values in myvalue per group

# select a subset of columns from a dataframe based on a substring or a custom list
newdf <- df %>% select(matches("BETA_"))
newdf <- df %>% select(all_of(mylistofcolumns)) # could also replace "all_of" with "any_of" if just want to keep any of the columns in mylist that are present in df

# Use a column name with special/annoying characters directly in filter/select
newdf <- filter(mydf,`P-VALUE`<5e-8)

# df group by a group but only keep one row from each group that meets a criteria (and breaks ties by picking first)
df <- as.data.frame(mydf %>% group_by(GENES) %>% filter(num_sig == max(num_sig)) %>% filter(1:n() == 1))

Code for various specific munging data tasks

library(dplyr)

# replace a single element in a df based on matching criteria
an <- as.data.frame(an)
an["Gene"][an["SNP"]=="rs1234"] <- "GENENAME1"

# turn rownames in df to a column
newdf <- as.data.frame((setDT(all, keep.rownames = TRUE)[]) %>% dplyr::rename(ID=rn)) # make rownames a column

# Separate a column that has a list (or any seperator) on the separator to split into either separate rows or separate columns
long <- tidyr::separate_rows(mydf,"Genes",sep=",") # split into multiple rows
mydf2 <- mydf2 %>% tidyr::separate(V2,sep=":",into=c("CHR","POS","A1","A2")) # split into multiple columns

# Collapse a group into one line each and comma separate all entries for each group for a given row
newdf <- mydf %>% group_by(SNP,BETA,SE,P_BOLT_LMM,MetaboliteName) %>% arrange(GENE) %>% summarize(GENES=paste(GENE,collapse=",")) # one row per variant, list genes

# Order a df by custom list (in this case "mypathwaylist") then subset to just first occurance
df_ordered <- mydf %>% mutate(Gene_Category = factor(Gene_Category, levels = mypathwaylist)) %>% arrange(Gene_Category) # custom order pathways
mydf <- as.data.frame(df_ordered %>% group_by(Gene) %>% filter(1:n() == 1)) # filter to only keep first occurance of each gene

# Switch wide matrix of SNP, beta / se / pvalue for each metabolite to long form
mydf_long <- data.table::melt(mydf, measure.vars = patterns("^BETA_", "^SE_", "^P_BOLT_LMM"),variable.name=c("Metabolite"), value.name=c("BETA", "SE", "P_BOLT_LMM"))
mydf_long$Metabolite <- rep(gsub("BETA_","",colnames(mylistofcolumns)),each=nmets)

# sort numerically a SINGLE list
l <- stringr::str_sort(l, numeric = TRUE)

Code for working with dynamic strings and variables

library(dplyr)

# Have R (such as during a for loop) treat your string saved as a variable as the column name for filter/select etc
for (col in 1:(ncol(mydf)-1)){
group_num=paste0("V",col,sep="")
group <- filter(mydf,!!sym(group_num) > 1) # filter to keep rows > 1 in the specified column
}

# Add a column with a dynamic (custum based on other variable) column name
i=1
varname <- paste0("cluster",i)
df[, varname] <- "test" # adds column called cluster1 to the dataframe

# Treat variable name as a string and/or turn a string into a variable
deparse(substitute(my_variable)) # variable name to string
eval(parse(text = my_string)) # string to variable name

Code for various specific reformating data tasks

library(dplyr)

# Replace the last occurance of something with whatever
gsub("l([^l]*)$", "hello", replacement="\\1") # replace "l" in this case with nothing
gsub("_([^_]*)$", p$V1, replacement=" \\1") # replace last underscore with a space

# Remove "chr" before chromosome numbers in the entire dataset (e.g. chr1 --> 1) and then add it back
df$CHROM <- as.integer(gsub("chr","",as.character(df$CHROM))) # remove "chr" before each
df$CHROM <- paste0("chr",df$CHROM) # add "chr" before each... (or could use gsub("^","chr",df$CHROM))

# Delete a symbol and all characters after it for all elements in a df column
mydf$genes <- gsub("\\..*","",mydf$genes)

Code for various miscellaneous tasks

library(dplyr)

# Use summarize to calculate mean and se for each group
newdf <- mydf %>% group_by(Pathway) %>% summarize(absvalue=abs(myvalues),avgvalue=mean(myvalues),sevalue=sd(myvalues)/sqrt(n()))

# Loop through a bunch of files and bind them all together with a new column that has a modified
# name indicating which file that row came from
my.path <- "/my/path/to/these/file/of/interest/" # set the directory path
my.list <- paste0(my.path, list.files(my.path, pattern = "_results.txt")) # create list of file names, assumes all files in this folder follow this pattern 
my.joined <- data.table::rbindlist(lapply(my.list, function(h) data.table::fread(h) %>% mutate(MetaboliteName=gsub(".*interest/*", "", gsub("_results.*", "", h))))) # Read in all files, bind them together and add a column with the name of the trait the file came from

# Labeling points in a plot
library(ggrepel)
library(ggplot2)
ggplot(mydf,aes(x=PA1,y=PA2,color=ColorColumn),size=6)+geom_point()+labs(x="Factor 1",y="Factor 2")+
	geom_label_repel(aes(label=Metabolites), size=4, show.legend = FALSE,force=10,segment.size=0.25)+
  scale_color_brewer(palette = "Dark2") + theme_classic()
ggsave(paste0(PLOT_DIR,prep,"_scores12.pdf"),height=15,width=20)

# globally for entire session set to always show all labels, regardless of too many overlaps, for ggrepel
options(ggrepel.max.overlaps = Inf)

# Quickly load a very large dataset
mydf <- data.table::fread("mylargedataset.tsv")

Oddly Specific, Potentially Useful Code - Data Analysis Unix snippets

2023-08-04T00:00:00-07:00

This is a collection of oddly specific, potential useful code snippets that I use frequently enough in my general data analysis work in unix that I wanted to put them in one place as a reference. Some of these things can be a bit of a pain to figure out or look up, so I wanted to share in case it helps someone else.

Each of the below code sets is completely independent from one another if under a separate comment (#), even within a given code block.

Fun things with for loops

# General for loop structure for custom list of variables
var=myfile1.txt
for met in var1 var2 var3 var4 var5 var6
do
    sumstats=myfile2.gz
    output=myoutputfile.gz
    zgrep -wFf ${var} ${sumstats} | gzip -c > ${output}
    echo $met
    echo $sumstats
    echo $output
done

# General for loop structure for piping the contents of a file into the for loop as variables    
while read p; do
echo $p
done <myfile.txt

# Rename all files in a directory, three different scenarios
for x in *.log; do y=$(sed 's/_/-/2' <<< "$x"); mv "$x" "$y"; done # with a certain extension by changing the second occurrence of hyphen to a dash
for x in *; do y=$(sed 's/all_metabolites/all_metabolites164/' <<< "$x"); mv "$x" "$y"; done # switch all files that start with all_metabolites to starting with all_metabolites164
for filename in *.txt.gz; do mv -i "$filename" "${filename#*_*_}"; done # delete everything before and including second underscore

# Move all files that meet a certain criteria in one folder to another
for x in myfileprefix*.gz; do y="myfileprefix/"; mv $x ${y}${x}; done

# Print out all file names for files that have over a certain number of lines
for i in *.txt
do
num=`wc -l $i | tr ' ' '\n' | head -n 1`
if (( num > 50)); then
    echo $i
fi
done
    
# Loop through pairwise combinations but not include repeat pairs (if have a-b don’t have b-a)
while read met1; do
  while read met2; do
    if [ "$met1" \< "$met2" ]
    then
     echo "Pairs $met1 and $met2"
    fi
    done <myfile1.txt
  done <myfile1.txt
    
# Loop through files and reformat some of the column header names and add new columns based on dividing/summing other columns
while read t; do
echo $t
file_name=${t}_myfilesuffix.txt.gz
in_file_path=/path/to/my/file/
in_file=${in_file_path}${file_name}
out_file_path=/path/to/write/file/to/
out_file=${out_file_path}${file_name}
sed -e '1s/rsid/SNP/' -e '1s/#CHR/CHR/' -e '1s/POS/BP/' -e '1s/ALT/A1/' -e '1s/REF/A2/' $in_file > temp.txt.gz # add custom header
zcat temp.txt.gz | awk 'BEGIN { FS = "\t" ; OFS = "\t"} ; {if(NR == 1) print $0,"Z","N"; else if(NR > 1) print $0, $7/$8,$12+$13}' > $out_file
rm temp.txt.gz
done <traits.txt
    
# Progressively add extra columns from dif files to one file
in_file=/my/path/filename1.tsv.gz
out_file=/my/output/path/filename2.tsv.gz
zcat ${in_file} | cut -f 1,7 > ${out_file}
while read t; do
echo $t
file_name=fileprefix.${t}.filesuffix.gz
in_file_path=/my/path/
in_file=${in_file_path}${file_name}
out_file2=/my/output/path/filename3.txt
zcat ${in_file} | cut -f 11,16 | sed -e '1s/BETA/BETA_'"${t}"'/' -e '1s/P_BOLT_LMM/P_BOLT_LMM_'"${t}"'/' | paste $out_file - > $out_file2 # add custom header to munged file
mv $out_file2 $out_file
done <filename4.txt
    
# Go through files with an extension and rename them to not have their extension, then gzip them
ls *.txt.gz | while read file ; do mv $file $(basename $file .gz); gzip $(basename $file .gz); done

Code for exploring and getting a feel for directories and files

# Find all subdirectories in the current directory that are empty
find . -mindepth 1 -maxdepth 1 -empty -type d

# Find all subdirectories in the current directory that are NOT empty
find . -mindepth 1 -maxdepth 1 -not -empty -type d

# Find all files with a certain substring in the filename
find . -name "*.HDL_L.*"

# Count how many directories have more than one file in them (can remove the | wc -l if want to view which they are) and such
find . -type f -printf '%h\n' | sort | uniq -d | wc -l
find . -type f -printf '%h\n' | sort | uniq -c # or see how many files each directory has
find . -type f -printf '%h\n' | sort | uniq -c | grep -w 1 # or search for which directories have a specific number of files (1 in this case)

# Count files + folders in a directory
ls | wc -l

Munging files and directories

# Remove all slurm files in a directory
find . -name "slurm*" -exec rm {} \; # can do `find . -name "slurm*"` first to see what will be deleted

# Cleanly list the difference between two files
diff metabolite_list.txt all_metabolites_list.txt | grep "<" | sed 's/^<//g'

# Filter file by another file
awk -F'\t' 'NR==FNR{a[$1];next} $1 in a' keep.tsv fullfile.tsv > filtered.txt # filters fullfile.tsv to only keep rows that match value in its col 1 with the keep.txt file's col 1

# Combine files
awk 'NR==FNR{A[$2]=$0; next} $3 in A {print A[$3], $0}' file2 file1 > newfile # This keeps the header. `A[$2]=$0` applies to file2, where A stores the entire file2, indexed by its second column. `$3 in A` checks if the third column of file1 is in A, if so, append the rows.
    
# Split the specified file into new files named by the value in the specified column (6 here)
awk -F'\t' '{print>$6}' allsig_metvarpairs_1e6_annotated.tsv

# Delete all files and directories (or just delete files if add -type f) from last x minutes (here 3) or days
find . -cmin -3 -delete # from last x minutes (here 3)
find . -type f -mtime +7 -name '*.log' -execdir rm -- '{}' + # delete all files in this directory from older than 7 days

# Add header to all files (can do *.txt if just want txt files; change from CHROM to Gene_names_allannotated with the header of choice)
sed -i '1i CHROM POS Effect_allele Other_allele P MetaboliteName SNP Consequence Gene_names_allannotated' *

# Make a new file with a specific row deleted based on a string in that line
sed '/rs10778203/d' ./Phenylalanine_levels > temp_phe_nors10778203

Intro R snippets

2023-07-24T00:00:00-07:00

This is a collection of intro R code snippets that I found helpful to use as a cheat sheet when first starting out in my general data analysis work in R.

Code for getting a feel for the data and environment

# Get a feel for the data
dim(my_data) # output the dimensions rows by columns
class(my_data) # display type of variable
rownames(my_data) # view row names
colnames(my_data) # view column names
head(my_data,x) # view first x rows
tail(my_data,x) # view last x rows
is.na(my_data) # sets TRUE for NAs and FALSE elsewhere

# Get a feel for the environment
help(thingneedhelpwith) # get help with a function/package
setwd(/path/to/set/as/working/dir) # sets working directory
ls() # print current stored variables
dir() # prints files in working directory
getwd() # prints path location of working directory

Loading, modifying, and saving data

# Loading data
df <- read.table('file.txt') # read in a file of any type
df <- data.table::fread('file.txt',header=T) # faster read in file
write.table(df, 'file.txt') # write dataframe to a file of any type
load('file.RData') # load in R data file
save(df, file = 'file.Rdata') # save dataframe as an R data file
rm(list=ls()) # clears variables stored
INPUT_FILE=commandArgs(TRUE)[1] # example of accepting input from command line, change number based on how many inputs and which one referring to

# Loading key packages I use a lot
library(tidyr)
library(dplyr)
library(data.table)
library(RColorBrewer)
library(ggplot2)

# Changing data type
as.data.frame(my_data) # convert to a dataframe
as.data.table(my_data) # convert to a datatable
as.integer(my_data) # convert to integers
as.numeric(my_data) # convert to numbers
as.character(my_data) # convert to characters
cut(x, breaks = 4) #Turn a numeric vector into a factor but ‘cutting’ into sections

# Saving df as file
OUTPUT_FILE="/the/path/to/the/file/here/nameoffile.tsv"
write.table(df, OUTPUT_FILE, quote=F, sep="\t", row.names=F, col.names=T)

Plotting with ggplot

library(ggplot2)
# SCATTER PLOT: with each point labeled, manual color setting to correspond to metabolite group color key, changing axis label size and rotating x axis-labels only
# Changing labels for axes and color legend, showing all labels even if overlap and drawing line to point if label needs to be farther away
options(ggrepel.max.overlaps = Inf)
colors <- c("orange", "green", "red","purple", "blue", "pink")
mets <- c("Amino Acid","Fatty Acid", "Glycolysis Related Metabolite", "Ketone", "Lipid Particle", "Other")
ggplot(pathsOI,aes(x=pathway1.bed,y=pathway2.bed,color=Color_Group))+
	geom_point()+geom_label_repel(aes(label=metabolite), size=4, alpha=0.75, show.legend = FALSE,force=10,segment.size=0.25)+
	scale_color_manual(labels=mets, values = colors)+
	labs(x="Label for the x axis",y="Label for the y axis",color="Metabolite Group")+
	theme_classic()+ theme(axis.text=element_text(size=12),axis.title=element_text(size=14),axis.text.x = element_text(angle = 70, hjust = 1))
ggsave(paste0(PLOT_DIR,"nameofmyfigure.png"),height=12,width=12)

# VIOLIN PLOT: changing the violin width, transparancy and color + label individual points
ggplot(h_annot,aes(x=col1,y=col2,fill=Color_Group))+geom_violin(alpha=0.5)+ geom_boxplot(width=0.1)+
	geom_point()+geom_label_repel(aes(label=metabolite), size=4, alpha=0.75, show.legend = FALSE,force=10,segment.size=0.25)+
	scale_fill_manual(labels=mets, values = colors)+
	labs(x="Name for x axis",y="Name for y axis",fill="Metabolite Group")+
	theme_classic()+ theme(axis.text=element_text(size=12),axis.title=element_text(size=14),axis.text.x = element_text(angle = 70, hjust = 1))
ggsave(paste0(PLOT_DIR,"nameofmyfigure.png"),height=12,width=12)

# MATRIX CORRELATION-LIKE PLOT: plot cooccurance matrix visually
v[upper.tri(v,diag=FALSE)]<- NA # half to NAs
library(tidyr)
library(ggplot2)
cm <- as.data.frame(v) %>% mutate(Var1 = factor(row.names(.), levels=row.names(.))) %>%
  gather(key = Var2, value = value, -Var1, na.rm = TRUE, factor_key = TRUE)
cm <- cm %>% rename(Pathways1=Var1,Pathways2=Var2)
ggplot(data = cm) + geom_tile(aes(Pathways1, Pathways2, fill = value))+
      geom_text(aes(Pathways1, Pathways2, label = value))+theme_classic()+
theme(axis.text.x = element_text(angle = 70, hjust = 1))+ # text = element_text(size=20),
 scale_fill_gradient2(low = "red", high = "blue", mid = "white",
   midpoint = 0, name="Number of variants\nin both")+ # the "\n" adds a newline in the label
 coord_fixed()+labs(x="Pathways 1",y="Pathways 2")
ggsave("nameofmyfigure.png")