Today, I finished Computing and did exercises of Chapter 8.
Computing
# 5. Boosted Trees (gbm: gradient boosting machines)
library(gbm)
gbmmodel=gbm.fit(solTrainXtrans, solTrainY, distribution = "gaussian")
gbmmodel=gbm(y~., data=trainData, distribution = "gaussian")
# The furthest you can go is to split each node until there is only 1 observation in each terminal node.
# This would correspond to n.minobsinnode=1.
gbmgrid=expand.grid(.interaction.depth=seq(1, 7, by=2), .n.minobsinnode=10,
.n.trees=seq(100, 1000, by=50), .shrinkage=c(0.01, 0.1))
set.seed(100)
gbmtune=train(solTrainXtrans, solTrainY, method="gbm", tuneGrid = gbmgrid, verbose=FALSE)
gbmtune
system.time(gbmtune)
# 6. Cubist
library(Cubist)
# an argument, committees, fits multiple models
cubistmodel=cubist(solTrainXtrans, solTrainY, committees = 5)
cubistmodel
# an argument, neighbors,can take on a single integer value (0-9) to
# adjust the rule-based predictions from the training set
cubistpred=predict(cubistmodel, solTestXtrans)
summary(cubistpred)
head(cubistpred)
# the train function in the caret package can tune the model over values of
# committees and neighbors through resampling
cubisttuned=train(solTrainXtrans, solTrainY, method="cubist")
cubisttuned
Exercises
library(mlbench)
set.seed(200)
simulated=mlbench.friedman1(200, sd=1)
simulated=cbind(simulated$x, simulated$y)
simulated=as.data.frame(simulated)
colnames(simulated)[ncol(simulated)]="y"
head(simulated)
library(randomForest)
library(caret)
model1=randomForest(y~., data=simulated, importance=TRUE, ntree=1000)
rfimp1=varImp(model1, scale=FALSE)
rfimp1
simulated$duplicate1=simulated$V1+rnorm(200)*0.1
cor(simulated$duplicate1, simulated$V1)
model2=randomForest(y~., data=simulated, importance=TRUE, ntree=1000)
rfimp2=varImp(model2, scale=FALSE)
rfimp2
library(party)
model3ctr=cforest_control(mtry=ncol(simulated)-1)
model3tree=cforest(y~., data=simulated, controls = model3ctr)
model3tree
cfimp=varimp(model3tree)
cfimp
Tomorrow, I will continue to do exercises of Chapter 8.
11/09/2016
11/08/2016
Chapter 8 Computing
Today, I did Computing of Chapter 8.
Summary
# the R packages used in this section are caret, Cubist, gbm, ipred, party, partykit,
# randomForest, rpart, RWeka
# 1. Single Trees
# formula method
library(rpart)
rparttree=rpart(y~., data=trainData)
library(party)
ctreetree=ctree(y~., data=trainData)
library(caret)
set.seed(100)
rparttune1=train(solTrainXtrans, solTrainY, method="rpart2", tuneLength=12, trControl=trainControl(method="cv"))
rparttune1
set.seed(100)
rparttune2=train(solTrainXtrans, solTrainY, method="rpart", tuneLength=10, trControl=trainControl(method="cv"))
rparttune2
?rpart.control
?ctree_control
plot(rparttune1)
plot(rparttune2)
library(partykit)
# convert the rpart object to a party object
rparttree2=as.party(rparttree)
plot(rparttree2)
# 2. Model Trees
library(RWeka)
# formula method
m5tree=M5P(y~., data=trainData)
m5rules=M5Rules(y~., data=trainData)
set.seed(100)
m5tune=train(solTrainXtrans, solTrainY, method="M5", trControl = trainControl(method = "cv"),
# M=10 is the minimum number of samples needed to further splits the data to be 10
control= Weka_control(M=10))
plot(m5tune)
# 3. Bagged Trees
library(ipred)
# bagging uses the formula interface and ipredbagg has the non-formula interface
baggedtree=ipredbagg(solTrainY, solTrainXtrans)
baggedtree=bagging(y~., data = trainData)
baggedtree2=as.party(baggedtree)
# mtry is equal to the number of predictors
bagctr1=cforest_control(mtry=ncol(trainData)-1)
baggedtree=cforest(y~., data=trainData, controls = bagCtr1)
# 4. Random Forest
library(randomForest)
rfmodel=randomForest(solTrainXtrans, solTrainY)
rfmodel=randomForest(y~., data=trainData)
plot(rfmodel)
# the default for mtry in regression is the number of prediction divided by 3
rfmodel2=randomForest(solTrainXtrans, solTrainY, importance = TRUE, ntrees=1000)
plot(rfmodel2)
importance(rfmodel2)
Tomorrow, I will continue to do computing of Chapter 8.
Summary
# the R packages used in this section are caret, Cubist, gbm, ipred, party, partykit,
# randomForest, rpart, RWeka
# 1. Single Trees
# formula method
library(rpart)
rparttree=rpart(y~., data=trainData)
library(party)
ctreetree=ctree(y~., data=trainData)
library(caret)
set.seed(100)
rparttune1=train(solTrainXtrans, solTrainY, method="rpart2", tuneLength=12, trControl=trainControl(method="cv"))
rparttune1
set.seed(100)
rparttune2=train(solTrainXtrans, solTrainY, method="rpart", tuneLength=10, trControl=trainControl(method="cv"))
rparttune2
?rpart.control
?ctree_control
plot(rparttune1)
plot(rparttune2)
library(partykit)
# convert the rpart object to a party object
rparttree2=as.party(rparttree)
plot(rparttree2)
# 2. Model Trees
library(RWeka)
# formula method
m5tree=M5P(y~., data=trainData)
m5rules=M5Rules(y~., data=trainData)
set.seed(100)
m5tune=train(solTrainXtrans, solTrainY, method="M5", trControl = trainControl(method = "cv"),
# M=10 is the minimum number of samples needed to further splits the data to be 10
control= Weka_control(M=10))
plot(m5tune)
# 3. Bagged Trees
library(ipred)
# bagging uses the formula interface and ipredbagg has the non-formula interface
baggedtree=ipredbagg(solTrainY, solTrainXtrans)
baggedtree=bagging(y~., data = trainData)
baggedtree2=as.party(baggedtree)
# mtry is equal to the number of predictors
bagctr1=cforest_control(mtry=ncol(trainData)-1)
baggedtree=cforest(y~., data=trainData, controls = bagCtr1)
# 4. Random Forest
library(randomForest)
rfmodel=randomForest(solTrainXtrans, solTrainY)
rfmodel=randomForest(y~., data=trainData)
plot(rfmodel)
# the default for mtry in regression is the number of prediction divided by 3
rfmodel2=randomForest(solTrainXtrans, solTrainY, importance = TRUE, ntrees=1000)
plot(rfmodel2)
importance(rfmodel2)
Tomorrow, I will continue to do computing of Chapter 8.
11/07/2016
Chapter 8 (8.5 8.6 8.7)
Today, I read Chapter 8 (8.5 8.6 8.7) of the book.
Summary
8.5 Random Forests
If we start with a sufficiently large number of original samples and a relationship between predictors and response that can be adequately modeled by a tree, then trees from different bootstrap samples may have similar structures to each other (especially at the top of the trees) due to the underlying relationship.
From a statistical perspective, reducing correlation among predictors can be done by adding randomness to the tree construction process.
After carefully evaluating these generalizations to the original bagging algorithm, Breiman constructed a unified algorithm called random forests.
Every model in the ensemble is then used to generate a prediction for a new ample and these m predictions are averaged to give the forest’s prediction.
Random forests’ tuning parameter is the number of randomly selected predictors, k, to choose from at each split. The practitioner must also specify the number of trees for the forest.
Compared to bagging, random forests is more computationally efficient on a tree-by-tree basis since the tree building process only needs to evaluate a fraction of the original predictors at each split, although more trees are usually required by random forests.
The ensemble nature of random forests makes it impossible to gain an understanding of the relationship between the predictors and the response.
Strobl et al. (2007) developed an alternative approach for calculating importance in random forest models that takes between-predictor correlations into account.
8.6 Boosting
AdaBoost algorithm
Gradient boosting machines
The basic principles of gradient boosting are as follows: given a loss function (e.g., squared error for regression) and a weak learner (e.g., regression trees), the algorithm seeks to find an additive model that minimizes the loss function.
When regression tree are used as the base learner, simple gradient boosting for regression has two tuning parameters: tree depth and number of iterations.
In random forests, all trees are created independently, each tree is created to have maximum depth, and each tree contributes equally to the final model. The trees in boosting, however, are dependent on past trees, have minimum depth, and contribute unequally to the final model.
A regularization strategy can be injected into the final line of the loop. Instead of adding the predicted value for a sample to previous iteration’s predicted value, only a fraction of the current predicted value is added to the previous iteration’s predicted value. This fraction is commonly
referred to as the learning rate and is parameterized by the symbol, λ. This parameter can take values between 0 and 1 and becomes another tuning parameter for the model.
Friedman updated the boosting machine algorithm with a random sampling scheme and termed the new procedure stochastic gradient boosting.
8.7 Cubist
Some specific differences between Cubist and the previously described approaches for model trees and their rule-based variants are:
• The specific techniques used for linear model smoothing, creating rules, and pruning are different
• An optional boosting—like procedure called committees
• The predictions generated by the model rules can be adjusted using nearby points from the training set data
y ̂_par=a×y ̂_k+(1-a)×y ̂_p
where y ̂_k is the prediction from the current model and y ̂_p is from parent model above it in the tree.
If the covariance is large, this implies that the residuals generally have the same sign and relative magnitude, while a value near 0 would indicate no (linear) relationship between the errors of the two models.
Let e_k be the collection of residuals of the child model and e_p be similar values for the parent model.
The smoothing coefficient used by Cubist is:
a=(Var[e_p ]-Cov[e_k,e_p])/(Var[e_p-e_k])
Var[e_p ] is proportional to the parent model’s RMSE.
The mth committee model uses an adjusted response
y_((m))^*=y-(y ̂_((m-1) )-y)
To tune this model, different numbers of committees and neighbors were assessed.
Summary
8.5 Random Forests
If we start with a sufficiently large number of original samples and a relationship between predictors and response that can be adequately modeled by a tree, then trees from different bootstrap samples may have similar structures to each other (especially at the top of the trees) due to the underlying relationship.
From a statistical perspective, reducing correlation among predictors can be done by adding randomness to the tree construction process.
After carefully evaluating these generalizations to the original bagging algorithm, Breiman constructed a unified algorithm called random forests.
Every model in the ensemble is then used to generate a prediction for a new ample and these m predictions are averaged to give the forest’s prediction.
Random forests’ tuning parameter is the number of randomly selected predictors, k, to choose from at each split. The practitioner must also specify the number of trees for the forest.
Compared to bagging, random forests is more computationally efficient on a tree-by-tree basis since the tree building process only needs to evaluate a fraction of the original predictors at each split, although more trees are usually required by random forests.
The ensemble nature of random forests makes it impossible to gain an understanding of the relationship between the predictors and the response.
Strobl et al. (2007) developed an alternative approach for calculating importance in random forest models that takes between-predictor correlations into account.
8.6 Boosting
AdaBoost algorithm
Gradient boosting machines
The basic principles of gradient boosting are as follows: given a loss function (e.g., squared error for regression) and a weak learner (e.g., regression trees), the algorithm seeks to find an additive model that minimizes the loss function.
When regression tree are used as the base learner, simple gradient boosting for regression has two tuning parameters: tree depth and number of iterations.
In random forests, all trees are created independently, each tree is created to have maximum depth, and each tree contributes equally to the final model. The trees in boosting, however, are dependent on past trees, have minimum depth, and contribute unequally to the final model.
A regularization strategy can be injected into the final line of the loop. Instead of adding the predicted value for a sample to previous iteration’s predicted value, only a fraction of the current predicted value is added to the previous iteration’s predicted value. This fraction is commonly
referred to as the learning rate and is parameterized by the symbol, λ. This parameter can take values between 0 and 1 and becomes another tuning parameter for the model.
Friedman updated the boosting machine algorithm with a random sampling scheme and termed the new procedure stochastic gradient boosting.
8.7 Cubist
Some specific differences between Cubist and the previously described approaches for model trees and their rule-based variants are:
• The specific techniques used for linear model smoothing, creating rules, and pruning are different
• An optional boosting—like procedure called committees
• The predictions generated by the model rules can be adjusted using nearby points from the training set data
y ̂_par=a×y ̂_k+(1-a)×y ̂_p
where y ̂_k is the prediction from the current model and y ̂_p is from parent model above it in the tree.
If the covariance is large, this implies that the residuals generally have the same sign and relative magnitude, while a value near 0 would indicate no (linear) relationship between the errors of the two models.
Let e_k be the collection of residuals of the child model and e_p be similar values for the parent model.
The smoothing coefficient used by Cubist is:
a=(Var[e_p ]-Cov[e_k,e_p])/(Var[e_p-e_k])
Var[e_p ] is proportional to the parent model’s RMSE.
The mth committee model uses an adjusted response
y_((m))^*=y-(y ̂_((m-1) )-y)
To tune this model, different numbers of committees and neighbors were assessed.
Tomorrow, I will start computing on Chapter 8.
11/05/2016
Chapter 8 (8.3 8.4)
Today, I read Chapter 8 (8.3 8.4).
Summary
Summary
8.3
Rule-Based Models
The
complexity of the model tree can be further reduced by either removing entire
rules or removing some of the conditions that define the rule.
In
figure 8.12, pruning has a large effect on the model and smoothing just has a
large impact on the unpruned models.
The
number of terms in the linear models decreases as more rules are created. This makes
sense because there are fewer data points to construct deep trees.
8.4
Bagged Trees
Next week, I will continue to read Chapter 8.
Bagging,
short for bootstrap aggregation, is a general approach that uses bootstrapping
in conjunction with any regression model to construct an ensemble.
Advantages:
1. Reduce
the variance and be more stable (average)
2. Provide
their own internal estimate of predictive performance that correlates well with
either cross-validation estimates or test set estimates (out-of-bag samples)
Most
improvement in predictive performance is obtained aggregating across ten
bootstrap replications.
Caveats:
1. Computational
costs and memory requirements increase as the number of bootstrap samples
increases. (parallel computing)
2. A
bagged model is less interpretable than a model that is not bagged. (variable
importance)
11/02/2016
Chapter 8 (8.1 8.2)
Today, I read Chapter 8 (8.1 8.2).
Summary
Once the tree has been finalized, we begin to assess the relative importance of the predictors to the outcome.
If SSE is the optimization criteria, then the reduction in the SSE for the training set is aggregated for each predictor.
The model tends to rely more on continuous predictors than the binary ones.
Unbiased Regression Tree Techniques:
GUIDE: generalized, unbiased, interaction detection and estimation algorithm (decouple the process of selecting the split variable and the split value)
Conditional Inference Trees: statistical hypothesis tests are used to do an exhaustive search across the predictors and their possible split points.
For a candidate split, a statistical test is used to evaluate the difference between the means of the two groups created by the split and a p-value can be computed for the test.
Regression Model Trees
M5
The splitting criterion is different.
The terminal nodes predict the outcome using a linear model (as opposed to the single average).
When a sample is predicted, it is often a combination of the predictions from different models along the same path through the tree.
The main implementation of this technique is a “rational reconstruction” of this model call M5.
Split criterion:
reduction=SD(S)-∑_(i=1)^P▒〖n_i/n×SD(S_i)〗
n_i is the number of samples in partition i.
The split that is associated with the largest reduction in error is chosen and a linear model is created within the partitions using the split variable in the model.
Once the complete set of linear models have been created, each undergoes a simplification procedure to potentially drop some of the terms.
Adjusted Error Rate=n^*+pn*-pi=1n*yi-yi
n^* is the number of training set data points that were used to build the model and p is the number of parameters.
Model trees also incorporate a type of smoothing to decrease the potential for over-fitting. The technique is based on the “recursive shrinking” methodology.
The two predictions are combined using
y_((p) )=(n_((k)) y ̂_((k) )+cy ̂_((p) ))/(n_((k))+c)
(the equation on page 185 may be wrong)
y ̂_((k) ) is the prediction from the child node, n_((k)) is the number of training set data points in the child node, y ̂_((p) ) is the prediction from the parent node, and c is a constant with a default value of 15.
Pruning & Smoothing
Smoothing the models has the effect of minimizing the collinearity issues.
Summary
Once the tree has been finalized, we begin to assess the relative importance of the predictors to the outcome.
If SSE is the optimization criteria, then the reduction in the SSE for the training set is aggregated for each predictor.
The model tends to rely more on continuous predictors than the binary ones.
Unbiased Regression Tree Techniques:
GUIDE: generalized, unbiased, interaction detection and estimation algorithm (decouple the process of selecting the split variable and the split value)
Conditional Inference Trees: statistical hypothesis tests are used to do an exhaustive search across the predictors and their possible split points.
For a candidate split, a statistical test is used to evaluate the difference between the means of the two groups created by the split and a p-value can be computed for the test.
Regression Model Trees
M5
The splitting criterion is different.
The terminal nodes predict the outcome using a linear model (as opposed to the single average).
When a sample is predicted, it is often a combination of the predictions from different models along the same path through the tree.
The main implementation of this technique is a “rational reconstruction” of this model call M5.
Split criterion:
reduction=SD(S)-∑_(i=1)^P▒〖n_i/n×SD(S_i)〗
n_i is the number of samples in partition i.
The split that is associated with the largest reduction in error is chosen and a linear model is created within the partitions using the split variable in the model.
Once the complete set of linear models have been created, each undergoes a simplification procedure to potentially drop some of the terms.
Adjusted Error Rate=n^*+pn*-pi=1n*yi-yi
n^* is the number of training set data points that were used to build the model and p is the number of parameters.
Model trees also incorporate a type of smoothing to decrease the potential for over-fitting. The technique is based on the “recursive shrinking” methodology.
The two predictions are combined using
y_((p) )=(n_((k)) y ̂_((k) )+cy ̂_((p) ))/(n_((k))+c)
(the equation on page 185 may be wrong)
y ̂_((k) ) is the prediction from the child node, n_((k)) is the number of training set data points in the child node, y ̂_((p) ) is the prediction from the parent node, and c is a constant with a default value of 15.
Pruning & Smoothing
Smoothing the models has the effect of minimizing the collinearity issues.
Tomorrow, I will continue to read Chapter 8.
11/01/2016
Chapter 7 exercises
Today, I finished exercises in Chapter 7 and started reading Chapter 8.
Exercises in Chapter 7
#7.1
set.seed(100)
x=runif(100, min=2, max=10)
y=sin(x)+rnorm(length(x))*0.25
sindata=data.frame(x=x, y=y)
plot(x, y)
datagrid=data.frame(x=seq(2, 10, length=100))
library(kernlab)
rbfsvm=ksvm(x=x, y=y, data=sindata, kernel="rbfdot", kpar="automatic", C=1, epsilon=0.1)
#C: cost of constraints violation; epsilon: epsilon in the insensitive-loss function
#kpar=list(sigma=n), when n gets smaller, the line gets flatter. It can be kpar="automatic".
modelprediction=predict(rbfsvm, newdata=datagrid)
points(x=datagrid$x, y=modelprediction[,1], type="l", col="blue")
#type="p, l, b, c, o, s, h, n"
#7.2
library(mlbench)
set.seed(200)
#inputs are 10 independent variables uniformly distributed on the interval [0,1],
#only 5 out of 10 are actually used in the formula
trainingdata2=mlbench.friedman1(200, sd=1)
#convert 'x' data from a matrix to a data frame
trainingdata2$x=data.frame(trainingdata2$x)
featurePlot(trainingdata2$x, trainingdata2$y)
library(caret)
knnmodel=train(trainingdata2$x, trainingdata2$y, method = "knn", preProcess = c("center", "scale"), tuneLength = 10)
knnmodel
knnpred=predict(knnmodel, newdata=trainingdata2$x)
postResample(pred = knnpred, obs = trainingdata2$y)
#7.3
#build SVM, neural network, MARS and KNN mdoels
# 7.4
#if nonlinear models outperform the optimal linear model, it shows that there is a nonlinear relationship between
#predictors and outcomes. if there is mo significant outperformance, linear model will be recommended.
#7.5
#top important predictors of optimal linear model and optimal nonlinear model are different.
#top predictors which are unique to the optimal nonlinear model is proved to own definitely a nonlinear
#relationship with outcomes
Chapter 8
Regression Trees and Rule-Based Models
Weaknesses: 1. Model instability; 2. Less-than-optimal predictive performance
If the relationship between predictors and the response cannot be adequately defined by rectangular subspaces of the predictors, then tree-based or rule-based models will have larger prediction error than other kinds of models.
8.1
Basic Regression Trees
The predictor to split on and value of the split
The depth or complexity of the tree
The prediction equation in the terminal nodes
CART: classification and regression tree methodology
SSE finds the optimal split-point for every predictor
SSE=∑_(i∈S_1)▒〖(y_i-y ̅_1)〗^2 +∑_(i∈S_2)▒〖(y_i-y ̅_2)〗^2
Cost-complexity tuning
SSEC_p=SSE+Cp×(#Terminal Nodes)
C_p is the complexity parameter
Smaller penalties tend to produce more complex models, which result in larger trees.
Tomorrow, I will continue to read Chapter 8.
Exercises in Chapter 7
#7.1
set.seed(100)
x=runif(100, min=2, max=10)
y=sin(x)+rnorm(length(x))*0.25
sindata=data.frame(x=x, y=y)
plot(x, y)
datagrid=data.frame(x=seq(2, 10, length=100))
library(kernlab)
rbfsvm=ksvm(x=x, y=y, data=sindata, kernel="rbfdot", kpar="automatic", C=1, epsilon=0.1)
#C: cost of constraints violation; epsilon: epsilon in the insensitive-loss function
#kpar=list(sigma=n), when n gets smaller, the line gets flatter. It can be kpar="automatic".
modelprediction=predict(rbfsvm, newdata=datagrid)
points(x=datagrid$x, y=modelprediction[,1], type="l", col="blue")
#type="p, l, b, c, o, s, h, n"
#7.2
library(mlbench)
set.seed(200)
#inputs are 10 independent variables uniformly distributed on the interval [0,1],
#only 5 out of 10 are actually used in the formula
trainingdata2=mlbench.friedman1(200, sd=1)
#convert 'x' data from a matrix to a data frame
trainingdata2$x=data.frame(trainingdata2$x)
featurePlot(trainingdata2$x, trainingdata2$y)
library(caret)
knnmodel=train(trainingdata2$x, trainingdata2$y, method = "knn", preProcess = c("center", "scale"), tuneLength = 10)
knnmodel
knnpred=predict(knnmodel, newdata=trainingdata2$x)
postResample(pred = knnpred, obs = trainingdata2$y)
#7.3
#build SVM, neural network, MARS and KNN mdoels
# 7.4
#if nonlinear models outperform the optimal linear model, it shows that there is a nonlinear relationship between
#predictors and outcomes. if there is mo significant outperformance, linear model will be recommended.
#7.5
#top important predictors of optimal linear model and optimal nonlinear model are different.
#top predictors which are unique to the optimal nonlinear model is proved to own definitely a nonlinear
#relationship with outcomes
Chapter 8
Regression Trees and Rule-Based Models
Weaknesses: 1. Model instability; 2. Less-than-optimal predictive performance
If the relationship between predictors and the response cannot be adequately defined by rectangular subspaces of the predictors, then tree-based or rule-based models will have larger prediction error than other kinds of models.
8.1
Basic Regression Trees
The predictor to split on and value of the split
The depth or complexity of the tree
The prediction equation in the terminal nodes
CART: classification and regression tree methodology
SSE finds the optimal split-point for every predictor
SSE=∑_(i∈S_1)▒〖(y_i-y ̅_1)〗^2 +∑_(i∈S_2)▒〖(y_i-y ̅_2)〗^2
Cost-complexity tuning
SSEC_p=SSE+Cp×(#Terminal Nodes)
C_p is the complexity parameter
Smaller penalties tend to produce more complex models, which result in larger trees.
10/31/2016
Chapter 7 Computing
Today, I finished reading the first four parts of Chapter 7 and completed the computing part of it.
Computing:
library(caret)
library(earth)
library(kernlab)
library(nnet)
#R has a number of packages and functions for creating neural networks such as nnet, neural and RSNNS.
#1. neural networks
nnetfit=nnet(predictors, outcome, size=5, decay=0.01, linout = TRUE, trace = FALSE,
maxit = 500, MaxNWts = 5*(ncol(predictors)+1)+5+1)
#trace=F: reduce the amount of printed output
#maxit: number of iterations to find parameter estimates
#MaxNWts: number of parameters
#this creates a single model with 5 hidden units assuming that the data in predictors have been
#standardized to be on the same scale
nnetavg=avNNet(predictors, outcome, size=5, decay=0.01, repeats = 5, linout = TRUE,
trace = FALSE, maxit=500, MaxNWts = 5*(ncol(predictors)+1)+5+1)
predict(nnetfit, newdata)
predict(nnetavg, newdata)
toohigh=findCorrelation(cor(solTrainXtrans), cutoff = 0.75)
trainxnnet=solTrainXtrans[,-toohigh]
testxnnet=solTestXtrans[,-toohigh]
#define the candidate models to test
nnetgrid=expand.grid(.decay=c(0, 0.01, 0.1), .size=c(1:10), .bag=FALSE)
set.seed(100)
nnettune=train(solTrainXtrans, solTrainY, method="avNNet", tuneGrid = nnetgrid, trControl = ctr1,
preProcess = c("center", "scale"), linout = TRUE, trace = FALSE,
MaxNWts = 10*(ncol(predictors)+1)+10+1, maxit=500)
#2. multivariate adaptive regression splines
marsfit=earth(solTrainXtrans, solTrainY)
marsfit
summary(marsfit)
marsgrid=expand.grid(.degree=1:2, .nprune=2:38)
set.seed(100)
marstuned=train(solTrainXtrans, solTrainY, method="earth", tuneGrid = marsgrid,
trControl = trainControl(method = "cv"))
marstuned
marspredict=predict(marstuned, solTestXtrans)
head(marspredict)
#estimate the importance of each predictor in the MARS model
varImp(marstuned)
#3. Support Vector Machines
#in kernlab package, the ksvm function is available for regression models and a large number of kernel functions
#the radial basis function is the default kernel function
library(kernlab)
#if appropriate values o the cost and kernel parameters are known, the model can be fit as:
svmfit=ksvm(x=solTrainXtrans, y=solTrainY, kernel="polydot", kpar="automatic", C=1, epsilon=0.1)
#if the values are unknown, they can be estimated through resampling.
#in train, "svmRadial", "svmLinear" or "svmPoly" fit different kernels
svmrtuned=train(solTrainXtrans, solTrainY, method = "svmRadial", preProcess = c("center", "scale"),
tuneLength = 14, trControl = trainControl(method = "cv"))
svmrtuned
svmrtuned$finalModel
#4. k-nearest neighbors
knndescr=solTrainXtrans[, -nearZeroVar(solTrainXtrans)]
set.seed(100)
knntune=train(knndescr, solTrainY, method = "knn", preProcess = c("center", "scale"),
tuneGrid = data.frame(.k=1:20), trControl = trainControl(method = "cv"))
knntune
Tomorrow, I will do exercises on Chapter 7.
Computing:
library(caret)
library(earth)
library(kernlab)
library(nnet)
#R has a number of packages and functions for creating neural networks such as nnet, neural and RSNNS.
#1. neural networks
nnetfit=nnet(predictors, outcome, size=5, decay=0.01, linout = TRUE, trace = FALSE,
maxit = 500, MaxNWts = 5*(ncol(predictors)+1)+5+1)
#trace=F: reduce the amount of printed output
#maxit: number of iterations to find parameter estimates
#MaxNWts: number of parameters
#this creates a single model with 5 hidden units assuming that the data in predictors have been
#standardized to be on the same scale
nnetavg=avNNet(predictors, outcome, size=5, decay=0.01, repeats = 5, linout = TRUE,
trace = FALSE, maxit=500, MaxNWts = 5*(ncol(predictors)+1)+5+1)
predict(nnetfit, newdata)
predict(nnetavg, newdata)
toohigh=findCorrelation(cor(solTrainXtrans), cutoff = 0.75)
trainxnnet=solTrainXtrans[,-toohigh]
testxnnet=solTestXtrans[,-toohigh]
#define the candidate models to test
nnetgrid=expand.grid(.decay=c(0, 0.01, 0.1), .size=c(1:10), .bag=FALSE)
set.seed(100)
nnettune=train(solTrainXtrans, solTrainY, method="avNNet", tuneGrid = nnetgrid, trControl = ctr1,
preProcess = c("center", "scale"), linout = TRUE, trace = FALSE,
MaxNWts = 10*(ncol(predictors)+1)+10+1, maxit=500)
#2. multivariate adaptive regression splines
marsfit=earth(solTrainXtrans, solTrainY)
marsfit
summary(marsfit)
marsgrid=expand.grid(.degree=1:2, .nprune=2:38)
set.seed(100)
marstuned=train(solTrainXtrans, solTrainY, method="earth", tuneGrid = marsgrid,
trControl = trainControl(method = "cv"))
marstuned
marspredict=predict(marstuned, solTestXtrans)
head(marspredict)
#estimate the importance of each predictor in the MARS model
varImp(marstuned)
#3. Support Vector Machines
#in kernlab package, the ksvm function is available for regression models and a large number of kernel functions
#the radial basis function is the default kernel function
library(kernlab)
#if appropriate values o the cost and kernel parameters are known, the model can be fit as:
svmfit=ksvm(x=solTrainXtrans, y=solTrainY, kernel="polydot", kpar="automatic", C=1, epsilon=0.1)
#if the values are unknown, they can be estimated through resampling.
#in train, "svmRadial", "svmLinear" or "svmPoly" fit different kernels
svmrtuned=train(solTrainXtrans, solTrainY, method = "svmRadial", preProcess = c("center", "scale"),
tuneLength = 14, trControl = trainControl(method = "cv"))
svmrtuned
svmrtuned$finalModel
#4. k-nearest neighbors
knndescr=solTrainXtrans[, -nearZeroVar(solTrainXtrans)]
set.seed(100)
knntune=train(knndescr, solTrainY, method = "knn", preProcess = c("center", "scale"),
tuneGrid = data.frame(.k=1:20), trControl = trainControl(method = "cv"))
knntune
Tomorrow, I will do exercises on Chapter 7.
Subscribe to:
Posts (Atom)