title: Speeding up matrix row and column operations in R
body: <p>I have a positive big matrix:</p>
  
  <pre><code>set.seed(1)
  mat &lt;- matrix(abs(rnorm(130000*1000)),nrow=130000,ncol=1000)
  rownames(mat) &lt;- paste("r",1:nrow(mat),sep="")
  </code></pre>
  
  <p>The <code>rownames</code> of <code>mat</code> are associated with a <code>parent.id</code>:</p>
  
  <pre><code>row.ids.df &lt;- data.frame(row.id=rownames(mat),parent.id=paste("p",sort(sample(13000,130000,replace=T)),sep=""))
  </code></pre>
  
  <p>such that every few rows are associated with the same <code>parent.id</code>.</p>
  
  <p>I need to compute these operations for every <code>row</code> in <code>mat</code>:</p>
  
  <ol>
  <li><p><code>mean</code> of <code>log</code> of the row elements</p></li>
  <li><p><code>mean</code> proportion of of that row out of all rows with the same <code>parent.id</code></p></li>
  <li><p><code>mean</code> probit of the proportion of of that row out of all rows with the same <code>parent.id</code></p></li>
  <li><p><code>sd</code> probit of the proportion of of that row out of all rows with the same <code>parent.id</code></p></li>
  </ol>
  
  <p>Naturally this is the first solution that comes to mind:</p>
  
  <pre><code>require(VGAM)
  res.df &lt;- do.call(rbind,mclapply(1:nrow(mat), function(x) {
    idx &lt;- which(rownames(mat) %in% row.ids.df$row.id[which(row.ids.df$parent.id == row.ids.df$parent.id[which(row.ids.df$row.id == rownames(mat)[x])])])
    data.frame(mean.log=mean(log(mat[x,])),
               mean.proportion=mean(mat[x,]/apply(mat[idx,],2,sum)),
               mean.probit=mean(probit(mat[x,]/apply(mat[idx,],2,sum))),
               sd.probit=sd(probit(mat[x,]/apply(mat[idx,],2,sum))))
  }))
  </code></pre>
  
  <p>But I'm wondering if there's any way to achieve this faster.</p>
  
  <p>P.S.</p>
  
  <p>I don't think using <code>data.table</code> instead of matrix is the way to go:</p>
  
  <pre><code>require(data.table)
  require(microbenchmark)
  require(VGAM)
  
  set.seed(1)
  mat &lt;- data.table(matrix(abs(rnorm(13*5)),nrow=13,ncol=5))
  rownames(mat) &lt;- paste("r",1:nrow(mat),sep="")
  row.ids.df &lt;- data.frame(row.id=rownames(mat),parent.id=paste("p",sort(sample(2,13,replace=T)),sep=""))
  
  microbenchmark(df &lt;- do.call(rbind,lapply(1:nrow(mat), function(x) {
    idx &lt;- which(rownames(mat) %in% row.ids.df$row.id[which(row.ids.df$parent.id == row.ids.df$parent.id[which(row.ids.df$row.id == rownames(mat)[x])])])
    data.frame(mean.log=mean(as.numeric(log(mat[x,]))),
               mean.proportion=mean(as.numeric(mat[x,])/apply(mat[idx,],2,sum)),
               mean.probit=mean(probit(as.numeric(mat[x,])/apply(mat[idx,],2,sum))),
               sd.probit=sd(probit(as.numeric(mat[x,])/apply(mat[idx,],2,sum))))
  })))
  
  
  expr
   df &lt;- do.call(rbind, lapply(1:nrow(mat), function(x) {     idx &lt;- which(rownames(mat) %in% row.ids.df$row.id[which(row.ids.df$parent.id ==          row.ids.df$parent.id[which(row.ids.df$row.id == rownames(mat)[x])])])     data.frame(mean.log = mean(as.numeric(log(mat[x, ]))), mean.proportion = mean(as.numeric(mat[x,          ])/apply(mat[idx, ], 2, sum)), mean.probit = mean(probit(as.numeric(mat[x,          ])/apply(mat[idx, ], 2, sum))), sd.probit = sd(probit(as.numeric(mat[x,          ])/apply(mat[idx, ], 2, sum)))) }))
        min       lq     mean   median       uq     max neval
   65.08929 66.49415 69.78937 67.70534 70.38044 206.017   100
  &gt;
  </code></pre>
  
  <p>Compared to:</p>
  
  <pre><code>set.seed(1)
  mat &lt;- matrix(abs(rnorm(13*5)),nrow=13,ncol=5)
  rownames(mat) &lt;- paste("r",1:nrow(mat),sep="")
  row.ids.df &lt;- data.frame(row.id=rownames(mat),parent.id=paste("p",sort(sample(2,13,replace=T)),sep=""))
  
  require(VGAM)
  microbenchmark(df &lt;- do.call(rbind,lapply(1:nrow(mat), function(x) {
    idx &lt;- which(rownames(mat) %in% row.ids.df$row.id[which(row.ids.df$parent.id == row.ids.df$parent.id[which(row.ids.df$row.id == rownames(mat)[x])])])
    data.frame(mean.log=mean(log(mat[x,])),
               mean.proportion=mean(mat[x,]/apply(mat[idx,],2,sum)),
               mean.probit=mean(probit(mat[x,]/apply(mat[idx,],2,sum))),
               sd.probit=sd(probit(mat[x,]/apply(mat[idx,],2,sum))))
  })))
  
  
  Unit: milliseconds
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              expr
   df &lt;- do.call(rbind, lapply(1:nrow(mat), function(x) {     idx &lt;- which(rownames(mat) %in% row.ids.df$row.id[which(row.ids.df$parent.id ==          row.ids.df$parent.id[which(row.ids.df$row.id == rownames(mat)[x])])])     data.frame(mean.log = mean(log(mat[x, ])), mean.proportion = mean(mat[x,          ]/apply(mat[idx, ], 2, sum)), mean.probit = mean(probit(mat[x,          ]/apply(mat[idx, ], 2, sum))), sd.probit = sd(probit(mat[x,          ]/apply(mat[idx, ], 2, sum)))) }))
        min      lq     mean median       uq      max neval
   10.15047 10.2894 10.69573 10.428 10.69741 14.56724   100
  </code></pre>
  
  <p>Unless applying <code>as.numeric</code> each time I want to run an operation on <code>data.table</code> row is a bad idea.</p>