###########################################################
#    ************************************************     #
#    Social Networks and Health Training Program 2018     #
#    ************************************************     #
###########################################################

#**************************************************************************#                            
#From Raw Data to Network Objects: Data Cleaning for Social Network Analysis
#**************************************************************************#

#Author: Maria Cristina Ramos - Duke University

##################################################################
# INTRODUCTION: DATA CLEANING FOR SOCIAL NETWORKS ANALYSIS (SNA) # 
##################################################################

#Start by clearing old data
rm(list = ls())

#Running a command line in R: place cursor in line and ctrl+enter (non MAC), cmd+enter(MAC) no need to highlight. You will move to next line of code.
#If you do not want to move down a line alt+enter
#If you want to execute just a piece of the line, highlight only that piece
#and ctrl+enter
#use # for commenting
#to comment more than one line at a time, highlight the code, ctrl+shift+C at #the same time.

#Data cleaning for SNA: planning and executing a series of tasks 
#that transform raw data into objects that SNA tools will be able 
#to analyze. 

#Statnet and igraph are commonly used network analysis tools for R.  
#SNA using statnet or igraph often starts with the creation of a
#network object from scratch. 

#NOTE: statnet and igraph are incompatible. You cannot use both statnet and 
#igraph at the same time. If you want to use both of them, you have to do it 
#sequentially. Unload one so that you can load the other.
#to unload them use detach(package:statnet) or detach(package:igraph) 

#This tutorial uses statnet. The data cleaning process is essentially the 
#same for both statnet and igraph. 
#IMPORTANT: load statnet library(statnet) only once you are ready 
#to construct a network object. Otherwise it will mess up with some
#data inspection functions.

#To create a network object with statnet, we need our data to be
#in a certain format. 

#There are four possible formats from which statnet can construct a 
#network:

# - Adjacency matrix
m <- matrix(rbinom(25,1,0.5),5,5)
colnames(m) <- c("Jim", "Molly", "Liann", "Jo", "Jaemin")
rownames(m) <- colnames(m)
diag(m) <- 0
m

# - Edgelist 
elData<-data.frame(
  from_id=c("1","2","3","1","3","1","2"),
  to_id=c("1", "1", "1", "2", "2", "3", "3"),
  myEdgeWeight=c(1, 2, 1, 2, 5, 3, 9.5),
  stringsAsFactors=FALSE
)
elData

# - Incidence matrix
inci<-matrix(c(1,1,0,0, 0,1,1,0, 1,0,1,0),ncol=3,byrow=FALSE)
rownames(inci) <- c("Jim", "Molly","Dana", "Liann")
colnames(inci) <- c("e1", "e2","e3")
inci

# - Bipartite network
m <- matrix(rbinom(25,1,0.5),5,5)
rownames(m) <- c("Jim", "Molly", "Liann", "Jo", "Jaemin")
colnames(m) <- c("Baseball club", "Chorus", "Volunteering", "Debate club", "Writing group")
diag(m) <- 0
m

#plus list of nodes with attributes (features such as gender, race, etc.)

#This workshop outlines a series of systematic steps to reduce the 
#uncertainty that often comes with working with R and make the 
#process easier! 

#From paralyzed to deliberate!

################
# THE PROCESS  # 
################

#1. Inspect/Evaluate Raw Data
#2. Make a Plan
#3. Construct Network Object
#4. Check your Work 

###########################################
#   PHASE 1: INSPECT/EVALUATE YOUR DATA   #
###########################################

# Goal: identify your data manipulation needs.

# a) Identify useful bits for network objects 
#   - What pieces will be the node ids?
#   - What pieces can be node attributes?
#   - What pieces will be the edges?
#   - What pieces can be edge attributes?   

# b) Identify issues we have to deal with  
#   - Issues at the observation level
#        -Extreme, nonsensical, and/or missing values. 
#   - Issues at the structure level
#        - Unnecesary columns: columns that do not contain useful info for 
#          our analysis.
#        - Preview rows (Qualtrics): apparent responses that are actually
#          you testing the survey. 
#        - Rows/columns in the wrong format: columns that should be rows,
#          rows that should be columns. Columns that should be combined, etc.      

# TIP: Don't start manipulating (moving, joining, etc.) anything until you 
# took a good look at your data. Get a good sense of what you have first.                 
#In the long run, it will be more efficient.

###########################################
#         PHASE 2: MAKE A PLAN            #
###########################################

#   Goal: come up with a task or series of tasks to solve each issue. No code
#   involved yet, just your plan. This will make the subsequent coding tasks
#   way easier.

# a) Plan changes at the observation level
#     - Remove missing values? Impute missing values? Recode? 
#       This decision depends on the source of the problem (e.g. a coding 
#       error versus a truly extreme value).

# b) Plan tasks for restructuring the dataset 
#     - First we need to choose the type of input structure for 
#        network objects:
#           - Adjacency matrix
#           - Edgelist 
#           - Incidence Matrix
#           - Bipartite Network
#           - + Nodelist

#     - Then, describe series of tasks to restructure your current 
#       dataset into the network object input (your roadmap) 

###########################################
#   PHASE 3: CONSTRUCT NETWORK OBJECT     #
###########################################

# Goal: Implement your plan

# You implement your plan by finding a function that will 
# do one or more of the tasks in your plan. 

# KEY: Having ALREADY identified the type of object/data 
# you need to manipulate and what task you want to do with 
# it will make it easy to find a suitable function for your tasks. 

#For example, I want to combine columns. Object: columns; task: combine 
#Having identified these I can now google for a function that combines columns.
#I can now go look in a package created to manipulate columns.
#Before I was just paralized not knowing what to do.               

# There are packages for specific types of objects/data 
# (e.g. stringr for strings, tidyr for rows and columns)

# Where can you find suitable functions for your tasks?

#a) Cheat sheets!

#     https://www.rstudio.com/resources/cheatsheets/ 
#     see also our dropbox folder for relevant cheat sheets

#     Advantages of cheat sheets:
#       - You build familiarity with the package.
#       - They have visuals! Easy to see what the function will do. 
#       - Quick, big picture of what is available. 
# 
# b) Package vignettes or manuals

#vignette(package = "dplyr") #list of topics for which there are vignettes about dplyr
#vignette("dplyr", package = "dplyr") #first argument "dplyr" was listed as Introduction to dplyr in the list of topics, so we call for it. 

#     https://cran.r-project.org/web/packages/network/network.pdf
#     this is the package we will use to create our network object.

# c) Googling it

#################################
#   PHASE 4: CHECK YOUR WORK    #
#################################

# Goal: Make sure the new network object **makes sense**

#Look at basic descriptives of the network and see if they fit with
#what we know about our raw data. 

#Now let's go through each phase in more detail by working with an example.  

#############################################
#   EXAMPLE: DATAMED - Nomination Network   #
#############################################

#Installing and loading packages
#install.packages("openxlsx") #I think this one you have to install.
#either comment or remove the line of code that installs a package right 
#after installing it.
library(openxlsx)
library (dplyr)
library (tidyr)
library(stringr)

#Importing Data
#This is a synthetic (fake) dataset I created for the purposes of this lab.
#We have two files. The first file contains responses to a 
#qualtrics survey. The second file is a person codebook. 

#"Respondents" are medical professionals and were asked to list up to ten
#other professionals with whom they discuss professional matters. 

#File 1: Field data from Qualtrics
setwd("/Users/mariacristinaramos/Dropbox/Ongoing Projects/RA for Jim/Data Cleaning Workshop/")
list.files() #this lets you see what's in your working directory
datamed <- read.xlsx("datamed_raw.xlsx")

#File 2: Person Codebook
pcode <- read.xlsx("datamed_raw.xlsx", sheet = 2)

##################################
#   PHASE 1: INSPECT YOUR DATA   #
##################################

#2 Steps
#Step 1. Understand the Structure
#Step 2. Understand at the Observation level

#1. Understand the Structure of your Data
#########################################

#a) Check the class of the dataset. 
class(datamed)
class(pcode)
#You can use class with any type of R object
#Most data come in tabular format, but still it is good to check.

#We can already see that we have more nodes in our person codebook (225)
#than responses to the survey (217). 

#b) view the column names to get a first sense of what you have. 
names(datamed)
names(pcode)
#RED FLAG: some of the column names are not very informative.

#We look at the survey and find out that
#Q32 = R's Name
#Q33 = R's Last Name
#Q.1 - Q.10 = R's Nominees 
#Q34 = R's Gender
#Q35 = R's Age
#Q36 = R's Area within Medicine
#Q37 = R's Hospital Affiliation

#c) See a compact summary of the data
str(datamed)
glimpse(datamed) #dplyr's version of str()
str(pcode)
#str stands for structure

#The str() function tells you the dimensions of the data. 
#In addition, the str () function tells you the class and 
#first observations of each variable.

#The str() function is particularly useful to identify bits of
#information you can use in your analysis. 
#In conjunction with your codebook or survey, you can identify 
#variables that contain information that we can use for 
#constructing our network. 

#str() is also useful to identify NAs
#Note R encodes missing values as NAs. Look for other common 
#missing values encodings in your data (-1, 9999, N/A).

#RED FLAGS: 
#unnecessary columns
#previews in Distribution Channel. 
#R's name and last name in different columns.
#R's name and last name (IDs) are strings, nominee IDs are numbers
#NAs in attribute variables 

#d) look the first and last rows 

#head() lets you look the first rows of a dataframe.
head(datamed) #first 6 rows (6 rows is the default)
#head(datamed, n=10)#first 10 rows. The n= tells R how many rows
#you would like to see
head(pcode)

#tail() lets you look at the last rows of the dataframe
tail(datamed) #last 6 rows
#tail(datamed, n=10) # set it to show you the last 10 rows.
tail(pcode)

#these functions let you see your dataset without 
#clutering the console. This is useful to detect
#whether your dataset is sorted in a particular way and to get a 
#deeper understanding of what each variable contains. 

#NOTICE: fewer NAs in attribute vars in tail. Could it be that those previews 
#are the NAs?

#e) Summary of the distribution of each variable
summary(datamed)

#For numeric variables, this means looking at means, 
#quartiles (including the median), and extreme values. 

#Summary will produce different summaries 
#depending on whether you are dealing with a character
#or factor variable. 

##Summary helps reveal unusual or 
#extreme values, missing values, special characters,etc. 
#Summary tells you the number of NAs in each variable 
#in the dataset.

#Also useful for assessing whether you will have enough
#variation in some variable to effectively use it as 
#an attribute for your network objects


#RED FLAG: If we know the max. node_id is 227, we shouldn't have values  
#greater than 227 in the nominations. However, we see that Q.1 has
#a max value of 609. 

#2. Look more closely at Observations
#####################################

#Should do this for each var of interest

#a) Tables for each variable of interest

#Important to find unusual observations, special characters, NAs, and
#variation in categories to see which attributes could be more distinguishing #features. 

#simple table
table(datamed$DistributionChannel)
#we have 5 preview observations

table(datamed$Q34, useNA="ifany") 
#IMPORTANT: $ selects a specific variable from the dataset. 
#useNA="ifany" includes NAs in your table

table(datamed$Q36, useNA = "ifany")
#RED FLAG: those Unknowns in Q36... 11 Unknowns

table(datamed$Q37, useNA="ifany")
#RED FLAG: only 3 people from City Hospital. Probably not a great distinguishing  
#attribute. 

#crosstab
with(datamed, table(Q36, Q37, useNA = "ifany"))

#b) Visualization 
hist(datamed$Q35)#histogram
boxplot(datamed$Q35) #boxplots are good for detecting outliers.

#ok. There are outliers. How do we find them?
which.max(datamed$Q35) #Which row contains the max value for Q35

which(datamed$Q35>60) #which rows contain values above 60 for Q35?

#see if we have any NAs in our person code
sum(is.na(pcode))

#We have looked at our data. 
#Now we have a sense of the data structure and the issues we need to deal #with. 
#We have a better sense of what variables will be useful to keep for our #network objects,how we should name those variables, and whether we have #missing data issues we should deal with. 

####################################
#   PHASE 2: MAKE A PLAN   #
####################################

#Remember: no code involved, just the plan. 

#A. Plan for dealing with unusual observations
##############################################

#When dealing with unusual values in your data, 
#you often must decide if they are just extreme or erroneous. 

#1. Remove preview observations and keep observations with NAs
#in attribute vars.

#2. Remove the 609 in Q.1

#3. Turn the "Unknown"s in Q36 into NAs

#B. Plan for constructing network
#################################

#1. Choose the type of input structure for network objects: 
#It seems like the easiest option is to go for an edgelist. This 
#might vary according to dataset.
#2. Describe series of tasks to restructure your current dataset into the network
#object input (your pseudo code):

#A. Fixing Node-id
#*****************

#Start by fixing the node_id issue since we will use that info in both our 
#node list and edgelist

#Step 1. Join name and last name in pcode file.
#Step 2. Join Q32 and Q33 (R's name and last name). 
#In this way we can match the two datasets with R's complete name.
#Step 3. Replace ID in our new nodelist dataframe with code from pcode. 

#B. Constructing nodelist
#************************

#Step 1. Create nodelist dataframe. 
#Step 2. Store new id, gender, area, and age cols in new dataframe.
#Step 3. Rename cols in nodelist dataframe 

#C. Constructing edgelist
#******************************

#Step 1. Create edgelist dataframe. 
#Step 2. Store new id and nominations in new dataframe.
#Step 3. move nominations from wide to long format
#Step 4. Remove NAs
#Step 5. Rename cols sender and target.

#3. Create network object using our nodelist and edgelist
#as input

#4. Add attributes

##########################################
#   PHASE 3: CONSTRUCT NETWORK OBJECTS   #
##########################################

#Every task you do follows the same process of data cleaning
#but in a small scale: you look at your data, you make a plan, you implement #it, and look again to check.

#1. Remove preview observations
names(datamed)
edgedata<- filter(datamed,DistributionChannel!="preview")
head(edgedata)#no more previews. Always, always check what you did.

#2. Remove the 609 in Q.1
edgedata$Q.1[edgedata$Q.1==609] <- NA
summary(edgedata)

#3. Turn the "Unknown"s in Q36 into NAs
edgedata$Q36[edgedata$Q36=="Unknown"] <- NA
table(edgedata$Q36, useNA = "ifany")

#A. Fixing Node-id
#*****************

#Step 1. Join name and last name in nodelist file.
#a useful function to join the contents of multiple columns?
#?unite
#TIP: use ?functionname when you know of a function that might work, 
#but you are not sure of what its arguments are

head(pcode)
pcode<- unite(pcode, "name", name:last_name, sep=" ", remove=TRUE)
head(pcode)

length(unique(pcode$name))#because we have 227 unique names, we 
#know no names were repeated

#sep="" very important. Tells R what should be between the pieces you 
#will unite
#Remove: whether the original columns you united should be removed or not. 

#Step 2. Join Q32 and Q33 (R's name and last name). 
str(datamed)
edgedata<- unite(edgedata,"name",Q32:Q33, sep=" ", remove=TRUE)
head(edgedata)

#Step 3. match name in new dataframe with name from node file. 
edgedata<- right_join(pcode,edgedata, by="name")
head(edgedata)
pcode[pcode$node_id==95,] #looking at whether 95 corresponds to Marilyn
#Hulett in the node file to see if our matching worked correctly.  

#We could have done this more efficiently by doing Step 2 and 3 together
#with pipes:
#edgedata<- unite(edgedata,"name",Q32:Q33, sep=" ", remove=TRUE)%>%
#right_join(pcode,edgedata, by="name")

#pipes are distinctive of dplyr. So you need to install the package. We did
#that at the beginning of the code.
#Pipes *pass* whatever is on the left to the right. This is different from #the usual R logic that goes from right to left.

#Usual R logic: a <- select(b,1:10) a equals a selection of b, its columns #from 1 to 10
#Pipes: a <- b%>%select(1:10). a equals: take b then select its columns 
#from 1 to 10.

#TIP: verbalizing your code can help. 

#B. Constructing nodelist
#************************
#Step 1. Create nodelist dataframe by joining our person codebook with the
#dataframe containing the attribute information. We need to fully join our 
#datasets instead of only using our datamed dataset because the datamed dataset
#does not contain all the nodes since not everyone replied to the survey. 
#Step 2. Store new id, gender, area, and age cols in new dataframe.
#Step 3. Rename cols in nodelist dataframe 
str(edgedata)

nodelist <- full_join(pcode,edgedata, by="node_id")%>%
  select(node_id, Q34:Q36)%>%
  rename(gender=Q34, age=Q35, area=Q36)

head(nodelist)

dim(na.omit(nodelist)) #To figure out from how many people you are missing 
#at least one attribute. 

#C. Constructing Edge list
#******************************

#Step 1. Create edgelist dataframe. 
#Step 2. Store new id and nominations in 
#new dataframe.
#Step 3. move nominations from wide to long format
#Step 4. Remove NAs
#Step 5. Rename sender and target.
str(edgedata)
edgedata <- select(edgedata, node_id, Q.1:Q.10)%>%
  gather(label, target, Q.1:Q.10, na.rm=TRUE)%>%
  select(-label)%>%
  rename(sender=node_id)%>%
  arrange(sender)
head(edgedata)

#### new steps (select) came given the function we used, but 
#having a plan made it easier to move forward

#3. Create network object using our nodelist edgelist
#as input
library(statnet)
mednet <- network.initialize(227, directed = TRUE)#network initialize
#is very useful to make sure you include isolates.
mednet <- network.edgelist(edgedata, mednet)
mednet<-network.edgelist(edgedata,network.initialize(227),ignore.eval=FALSE)

#4.Add node attributes

# Note: order of attributes you add must match vertex ids
# otherwise the attribute will get assigned to the wrong vertex

#This is how we see vertex names
mednet %v% "vertex.names"
#yep, in the correct order.

mednet %v% "gender" <- nodelist$gender
mednet %v% "age" <- nodelist$age
mednet %v% "area" <- nodelist$area

##################################
#   PHASE 4: CHECK YOUR WORK   #
##################################

#quick check
mednet
dim(edgedata)
#same number of edges

#checking attributes
summary.network(mednet)

#Now compare distributions of vertex atttributes 
head(nodelist)
#gender
table(datamed$Q34)
#area
table(datamed$Q36)
#age
hist(datamed$Q35)

plot(mednet)
#Do we see isolates?
#Does it make sense?

###########################################
#     TIPS: WHEN SOMETHING DOESN'T WORK   #
###########################################

#1. Try to understand the error message or look for clues in it.
#2. google the error.
#3. use help to see if the arguments you used are not right or not in the 
#right format
#4. use the package manual. toy examples and make substitutions until you 
#find what the function didn't like

#For more information
#?network #for information about creating networks using the network package. 
#?attribute.methods #for information about setting, modifying, or deleting network, vertex, or edge attributes. 

```