benchmarks.R (12896B)
1 source(file.path("build", "arma-benchmarks", "R", "arma.load.R")) 2 source(file.path("build", "arma-benchmarks", "R", "arma.load_events.R")) 3 source(file.path("R", "common.R")) 4 5 arma.load_benchmark_data <- function(attempt, framework, models, tags) { 6 data <- data.frame() 7 for (m in models) { 8 if (!(m %in% colnames(data))) { 9 data[,m] <- rep(NA, nrow(data)) 10 } 11 idx <- 1 12 for (t in tags) { 13 values <- arma.load( 14 file.path( 15 "build", "arma-benchmarks", "output", 16 "gpulab1", attempt, 10000, framework, m), 17 t, 18 ".*\\s+([0-9]+)us.*" 19 ) 20 name <- names(tags)[idx] 21 data[if (nchar(name) == 0) t else name,m] <- mean(values/1000/1000) 22 idx <- idx + 1 23 } 24 } 25 data 26 } 27 28 arma.print_openmp_vs_opencl <- function(model_names, row_names) { 29 options(asciiType="org") 30 models <- c("ar", "ma", "lh"); 31 frameworks <- c("openmp", "opencl") 32 tags <- list( 33 determine_coefficients=c("deteremine_coefficients", "determine_coefficients"), 34 validate=c("validate"), 35 generate_surface=c("generate_surface"), 36 nit=c("nit_acf", "nit_realisation"), 37 copy_to_host=c("copy_to_host"), 38 velocity=c("window_function", "second_function", "fft", "dev_to_host_copy"), 39 write_all=c("write_all") 40 ) 41 # remove names that were not requested 42 for (name in names(tags)) { 43 if (!(name %in% names(row_names))) { 44 tags[[name]] <- NULL 45 } 46 } 47 all_data <- list() 48 for (framework in frameworks) { 49 all_data[[framework]] <- arma.load_benchmark_data("a4", framework, models, tags) 50 } 51 # translate and pretty print in org-mode format 52 saved_row_names <- rownames(all_data$openmp) 53 for (framework in names(all_data)) { 54 rownames(all_data[[framework]]) <- 1:nrow(all_data[[framework]]) 55 } 56 big_table <- merge(all_data$openmp, all_data$opencl, by="row.names") 57 rownames(big_table) <- saved_row_names 58 big_table[,"Row.names"] <- saved_row_names 59 big_table <- big_table[,!(names(big_table) == "ma.y")] 60 big_table[,"Row.names"] <- sapply(big_table[,"Row.names"], function (c) get(c, row_names)) 61 colnames(big_table) <- sapply(colnames(big_table), function (c) get(c, model_names)) 62 # rownames(big_table) <- sapply(rownames(big_table), function (c) paste("~", c, "~", sep="")) 63 print(ascii(c("", "", "", "OpenMP", "", "OpenCL"))) 64 print(ascii(c("", "<r>", "<r>", "<r>", "<r>", "<r>"))) 65 arma.print_ascii(ascii(big_table, include.rownames=FALSE)) 66 } 67 68 arma.load_io_benchmark_data <- function(attempt, filesystems, suffix, tags) { 69 data <- data.frame() 70 for (fs in filesystems) { 71 if (!(fs %in% colnames(data))) { 72 data[,fs] <- rep(NA, nrow(data)) 73 } 74 idx <- 1 75 for (t in tags) { 76 values <- arma.load( 77 file.path( 78 "build", "arma-benchmarks", "output", "gpulab1", 79 paste(attempt, fs, suffix, sep="-"), 10000, "openmp", "ar"), 80 t, 81 ".*\\s+([0-9]+)us.*" 82 ) 83 name <- names(tags)[idx] 84 data[if (length(name) == 0 || nchar(name) == 0) t else name,fs] <- mean(values/1000/1000) 85 idx <- idx + 1 86 } 87 } 88 data 89 } 90 91 arma.print_sync_vs_async_io <- function(suffix_names, row_names, top_names) { 92 options(asciiType="org") 93 tags <- list("generate_surface", "write_all") 94 filesystems <- c("xfs", "nfs", "gfs") 95 all_data <- list() 96 for (suffix in c("seq", "par")) { 97 all_data[[suffix]] <- arma.load_io_benchmark_data("a5", filesystems, suffix, tags) 98 } 99 # translate and pretty print in org-mode format 100 saved_row_names <- rownames(all_data$seq) 101 for (suffix in names(all_data)) { 102 rownames(all_data[[suffix]]) <- 1:nrow(all_data[[suffix]]) 103 } 104 big_table <- merge(all_data$seq, all_data$par, by="row.names") 105 rownames(big_table) <- saved_row_names 106 big_table[,"Row.names"] <- saved_row_names 107 big_table[,"Row.names"] <- sapply(big_table[,"Row.names"], function (c) get(c, row_names)) 108 colnames(big_table) <- sapply(colnames(big_table), function (c) get(c, suffix_names)) 109 print(ascii(c("", "", "", top_names[[1]], "", "", top_names[[2]]))) 110 print(ascii(c("", "<r>", "<r>", "<r>", "<r>", "<r>", "<r>"))) 111 arma.print_ascii(ascii(big_table, include.rownames=FALSE)) 112 } 113 114 arma.plot_io_events <- function (names) { 115 fsnames <- list( 116 xfs="XFS", 117 nfs="NFS", 118 gfs="GlusterFS" 119 ) 120 filesystems <- c("xfs", "nfs", "gfs") 121 conf <- list( 122 a=list( 123 color='#000000', 124 lty="solid", 125 lwd=3, 126 name="generate_surface" 127 ), 128 b=list( 129 color='#9F2D20', 130 lty="solid", 131 lwd=3, 132 name="write_surface" 133 ) 134 ) 135 for (fs in filesystems) { 136 attempt <- paste("a5", fs, "events", sep="-") 137 data <- arma.load_events( 138 file.path("build", "arma-benchmarks", 139 "output", "gpulab1", attempt, 10000, "openmp", "ar"), 140 c("write_surface", "generate_surface", "programme") 141 ) 142 ev_prog <- data[data$event == "programme",] 143 ev_gen <- data[data$event == "generate_surface",] 144 ev_write <- data[data$event == "write_surface",] 145 ev_write$thread_no = max(ev_gen$thread_no) + 1 146 threads <- 0:max(ev_write$thread_no) 147 max_x <- max(ev_write$t1, ev_gen$t1)/1000/1000 148 plot.new() 149 plot.window(xlim=c(0,max_x), ylim=range(threads)) 150 conf$a$table=ev_gen 151 conf$b$table=ev_write 152 for (c in conf) { 153 table <- c$table 154 for (row in seq(1,nrow(table),1)) { 155 ev <- table[row,] 156 ys <- rep(ev$thread_no, 2) 157 xs <- c(ev$t0, ev$t1)/1000/1000 158 #points(xs[[1]], ys[[1]], pch=19, cex=0.4) 159 #arrows(xs[[1]], ys[[1]], xs[[2]], ys[[2]], angle=10, length=0.05) 160 lines(xs, ys, lwd=1, col=c$color) 161 points(xs, ys, col=c$color, cex=0.4) 162 } 163 } 164 axis(1, at=pretty(c(0,max_x))) 165 axis( 166 2, 167 at=threads, 168 labels=c(sapply( 169 threads[1:(length(threads)-1)], 170 function (t) paste("omp", t, sep="-") 171 ), "io-0"), 172 las=2 173 ) 174 mtext(names$xlab, side=1, line=3) 175 mtext(names$ylab, side=2, line=4) 176 title(fsnames[[fs]]) 177 } 178 legend( 179 "bottom", 180 inset=c(0,0.2), 181 legend=sapply(conf, function (c) c$name), 182 col=sapply(conf, function (c) c$color), 183 lty=sapply(conf, function (c) c$lty), 184 lwd=sapply(conf, function (c) c$lwd), 185 xpd=TRUE 186 ) 187 } 188 189 arma.load_realtime_data <- function () { 190 tags <- c( 191 "harts_g1", 192 "harts_g2", 193 "harts_fft", 194 "harts_copy_to_host" 195 ) 196 sizes <- 2^c(7:14) 197 frameworks <- c("openmp", "opencl") 198 attempt <- "a6" 199 data <- data.frame() 200 row <- 1 201 for (framework in frameworks) { 202 for (m in sizes) { 203 for (t in tags) { 204 all_data <- arma.load( 205 file.path("build", "arma-benchmarks", "output", "storm", attempt, m, framework), 206 t, 207 ".*\\s+([0-9]+)us.*" 208 ) 209 data[row,"framework"] <- framework 210 data[row,"size"] <- as.integer(m) 211 data[row,"t"] <- mean(all_data/1000/1000) 212 data[row,"routine"] <- t 213 row <- row + 1 214 } 215 } 216 } 217 data 218 } 219 220 arma.aggregate_by_size <- function (data, framework) { 221 fwdata <- data[data["framework"] == framework,] 222 fwdata <- aggregate( 223 fwdata$t, 224 by=list(size=fwdata$size), 225 FUN=sum 226 ) 227 fwdata <- setNames(fwdata, c("size", "t")) 228 fwdata 229 } 230 231 arma.filter_copy_to_host <- function (data) { 232 data[data["routine"] != "harts_copy_to_host",] 233 } 234 235 arma.add_text <- function (data, str, pos=4, off=1) { 236 len <- length(data$t) 237 text(data$size[[len-1]], data$t[[len-1]], str, pos=pos, offset=off) 238 } 239 240 arma.plot_realtime_data <- function (data, ...) { 241 args <- list(...) 242 openmp <- arma.aggregate_by_size(data, "openmp") 243 opencl <- arma.aggregate_by_size(data, "opencl") 244 opengl <- arma.aggregate_by_size(arma.filter_copy_to_host(data), "opencl") 245 plot.new() 246 plot.window( 247 xlim=range(c(openmp$size, opencl$size, opengl$size)), 248 ylim=range(ceiling(c(0, openmp$t, opencl$t, opengl$t)))) 249 lines(openmp$size, openmp$t, lty="solid", type="b") 250 lines(opencl$size, opencl$t, lty="dashed", type="b") 251 lines(opengl$size, opengl$t, lty="dotted", type="b") 252 axis(1, at=2^c(7:14)) 253 axis(2, at=c(0:8)) 254 box() 255 arma.add_text(openmp, "OpenMP", pos=3, off=1.5) 256 arma.add_text(opencl, "OpenCL", pos=3, off=1.5) 257 arma.add_text(opengl, "OpenCL/OpenGL", pos=3, off=-1.5) 258 } 259 260 arma.filter_by_framework_and_size <- function (data, size, framework) { 261 data <- data[data["framework"]==framework & data["size"] == size, ] 262 data <- data[c("routine", "t")] 263 rownames(data) <- c(1:length(data$t)) 264 data 265 } 266 267 arma.print_table_for_realtime_data <- function (data, routine_names, column_names) { 268 par(family="serif") 269 openmp <- arma.filter_by_framework_and_size(data, 2^14, "openmp") 270 opencl <- arma.filter_by_framework_and_size(data, 2^14, "opencl") 271 all_data <- merge(openmp, opencl, by="row.names") 272 all_data <- all_data[c("routine.x", "t.x", "t.y")] 273 all_data <- setNames(all_data, c("routine", "openmp", "opencl")) 274 # remove non-existent data copying 275 all_data[all_data$routine=="harts_copy_to_host", "openmp"] <- NA 276 all_data$routine <- sapply(all_data$routine, function (c) get(c, routine_names)) 277 all_data <- setNames(all_data, column_names) 278 print(ascii(c("", "<r>", "<r>"))) 279 arma.print_ascii(ascii(all_data, include.rownames=FALSE, digits=4)) 280 } 281 282 arma.load_bscheduler_data <- function (all_test_cases) { 283 all_data = data.frame( 284 framework=rep(NA,0), 285 size=rep(NA,0), 286 t=rep(NA,0) 287 ) 288 row <- 1 289 for (size in seq(10000, 30000, 2500)) { 290 for (test_case in all_test_cases) { 291 attempt <- test_case[[1]] 292 framework <- test_case[[2]] 293 hostname <- test_case[[3]] 294 data <- arma.load_events( 295 file.path( 296 "build", 297 "arma-benchmarks", 298 "output", 299 hostname, 300 attempt, 301 size, 302 framework, 303 "ar" 304 ), 305 c("programme") 306 ) 307 ev_prog <- data[data$event == "programme",] 308 all_data[row, 'framework'] <- paste(attempt, framework, sep="-") 309 all_data[row, 'size'] <- size 310 all_data[row, 't'] <- mean(ev_prog$t1 - ev_prog$t0)*1e-6 311 row <- row + 1 312 } 313 } 314 all_data 315 } 316 317 arma.load_bscheduler_performance_data <- function() { 318 all_test_cases <- list(c("a9-single-node-direct", "openmp", "m1"), 319 c("a9-single-node-direct", "bscheduler", "m1"), 320 c("a9-two-nodes-direct", "bscheduler", "m1")) 321 arma.load_bscheduler_data(all_test_cases) 322 } 323 324 arma.load_master_slave_failure_data <- function() { 325 all_test_cases <- list(c("a10-failure-direct-slave", "bscheduler", "m1"), 326 c("a9-single-node-direct", "bscheduler", "m1"), 327 c("a10-failure-direct-master", "bscheduler", "m1")) 328 arma.load_bscheduler_data(all_test_cases) 329 } 330 331 arma.plot_bscheduler_performance_data <- function (all_data, args) { 332 if (!('openmp_args' %in% names(args))) { 333 args$openmp_args = list(col="black", lty="solid", lwd=1, pch=20) 334 } 335 if (!('bsc1_args' %in% names(args))) { 336 args$bsc1_args = list(col="black", lty="dashed", lwd=1, pch=20) 337 } 338 if (!('bsc2_args' %in% names(args))) { 339 args$bsc2_args = list(col="black", lty="dotted", lwd=1, pch=20) 340 } 341 plot.new() 342 plot.window(xlim=range(all_data$size), ylim=range(0,all_data$t)) 343 conf <- list( 344 a=c(args$openmp_args, list( 345 framework='a9-single-node-direct-openmp', 346 name=args$openmp 347 )), 348 b=c(args$bsc1_args, list( 349 framework='a9-single-node-direct-bscheduler', 350 name=args$bsc1 351 )), 352 c=c(args$bsc2_args, list( 353 framework='a9-two-nodes-direct-bscheduler', 354 name=args$bsc2 355 )) 356 ) 357 for (c in conf) { 358 data <- all_data[all_data$framework==c$framework, ] 359 lines(data$size, data$t, col=c$col, lty=c$lty, lwd=c$lwd) 360 points(data$size, data$t, col=c$col, lwd=c$lwd, pch=c$pch) 361 } 362 legend( 363 "bottomright", 364 legend=sapply(conf, function (c) c$name), 365 col=sapply(conf, function (c) c$col), 366 lty=sapply(conf, function (c) c$lty), 367 lwd=sapply(conf, function (c) c$lwd) 368 ) 369 axis(1) 370 axis(2) 371 box() 372 } 373 374 arma.plot_master_slave_failure_data <- function (all_data, args) { 375 if (!('master_args' %in% names(args))) { 376 args$master_args = list(col="black", lty="solid", lwd=1, pch=20) 377 } 378 if (!('slave_args' %in% names(args))) { 379 args$slave_args = list(col="black", lty="dashed", lwd=1, pch=20) 380 } 381 if (!('nofailures_args' %in% names(args))) { 382 args$nofailures_args = list(col="black", lty="dotted", lwd=1, pch=20) 383 } 384 plot.new() 385 plot.window(xlim=range(all_data$size), ylim=range(0,all_data$t)) 386 conf <- list( 387 a=c(args$master_args, list( 388 framework='a10-failure-direct-master-bscheduler', 389 name=args$master 390 )), 391 b=c(args$slave_args, list( 392 framework='a10-failure-direct-slave-bscheduler', 393 name=args$slave 394 )), 395 c=c(args$nofailures_args, list( 396 framework='a9-single-node-direct-bscheduler', 397 name=args$nofailures 398 )) 399 ) 400 for (c in conf) { 401 data <- all_data[all_data$framework==c$framework, ] 402 lines(data$size, data$t, col=c$col, lty=c$lty, lwd=c$lwd) 403 points(data$size, data$t, col=c$col, lwd=c$lwd, pch=c$pch) 404 } 405 legend( 406 "bottomright", 407 legend=sapply(conf, function (c) c$name), 408 col=sapply(conf, function (c) c$col), 409 lty=sapply(conf, function (c) c$lty), 410 lwd=sapply(conf, function (c) c$lwd) 411 ) 412 axis(1) 413 axis(2) 414 box() 415 }