1# -*- coding: utf-8 -*- 2 3#------------------------------------------------------------------------- 4# drawElements Quality Program utilities 5# -------------------------------------- 6# 7# Copyright 2015 The Android Open Source Project 8# 9# Licensed under the Apache License, Version 2.0 (the "License"); 10# you may not use this file except in compliance with the License. 11# You may obtain a copy of the License at 12# 13# http://www.apache.org/licenses/LICENSE-2.0 14# 15# Unless required by applicable law or agreed to in writing, software 16# distributed under the License is distributed on an "AS IS" BASIS, 17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18# See the License for the specific language governing permissions and 19# limitations under the License. 20# 21#------------------------------------------------------------------------- 22 23from ctsbuild.common import * 24from ctsbuild.config import * 25from ctsbuild.build import * 26 27import os 28import sys 29import string 30import socket 31import fnmatch 32from datetime import datetime 33 34BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly")) 35BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build") 36BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs") 37BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs") 38 39EXECUTOR_PATH = "executor/executor" 40LOG_TO_CSV_PATH = "executor/testlog-to-csv" 41EXECSERVER_PATH = "execserver/execserver" 42 43CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data") 44 45COMPARE_NUM_RESULTS = 4 46COMPARE_REPORT_NAME = "nightly-report.html" 47 48COMPARE_REPORT_TMPL = ''' 49<html> 50<head> 51<title>${TITLE}</title> 52<style type="text/css"> 53<!-- 54body { font: serif; font-size: 1em; } 55table { border-spacing: 0; border-collapse: collapse; } 56td { border-width: 1px; border-style: solid; border-color: #808080; } 57.Header { font-weight: bold; font-size: 1em; border-style: none; } 58.CasePath { } 59.Pass { background: #80ff80; } 60.Fail { background: #ff4040; } 61.QualityWarning { background: #ffff00; } 62.CompabilityWarning { background: #ffff00; } 63.Pending { background: #808080; } 64.Running { background: #d3d3d3; } 65.NotSupported { background: #ff69b4; } 66.ResourceError { background: #ff4040; } 67.InternalError { background: #ff1493; } 68.Canceled { background: #808080; } 69.Crash { background: #ffa500; } 70.Timeout { background: #ffa500; } 71.Disabled { background: #808080; } 72.Missing { background: #808080; } 73.Ignored { opacity: 0.5; } 74--> 75</style> 76</head> 77<body> 78<h1>${TITLE}</h1> 79<table> 80${RESULTS} 81</table> 82</body> 83</html> 84''' 85 86class NightlyRunConfig: 87 def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []): 88 self.name = name 89 self.buildConfig = buildConfig 90 self.generator = generator 91 self.binaryName = binaryName 92 self.testset = testset 93 self.args = args 94 self.exclude = exclude 95 self.ignore = ignore 96 97 def getBinaryPath(self, basePath): 98 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath)) 99 100class NightlyBuildConfig(BuildConfig): 101 def __init__(self, name, buildType, args): 102 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args) 103 104class TestCaseResult: 105 def __init__ (self, name, statusCode): 106 self.name = name 107 self.statusCode = statusCode 108 109class MultiResult: 110 def __init__ (self, name, statusCodes): 111 self.name = name 112 self.statusCodes = statusCodes 113 114class BatchResult: 115 def __init__ (self, name): 116 self.name = name 117 self.results = [] 118 119def parseResultCsv (data): 120 lines = data.splitlines()[1:] 121 results = [] 122 123 for line in lines: 124 items = line.split(",") 125 results.append(TestCaseResult(items[0], items[1])) 126 127 return results 128 129def readTestCaseResultsFromCSV (filename): 130 return parseResultCsv(readFile(filename)) 131 132def readBatchResultFromCSV (filename, batchResultName = None): 133 batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename)) 134 batchResult.results = readTestCaseResultsFromCSV(filename) 135 return batchResult 136 137def getResultTimestamp (): 138 return datetime.now().strftime("%Y-%m-%d-%H-%M") 139 140def getCompareFilenames (logsDir): 141 files = [] 142 for file in os.listdir(logsDir): 143 fullPath = os.path.join(logsDir, file) 144 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"): 145 files.append(fullPath) 146 files.sort() 147 148 return files[-COMPARE_NUM_RESULTS:] 149 150def parseAsCSV (logPath, config): 151 args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath] 152 proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 153 out, err = proc.communicate() 154 return out 155 156def computeUnifiedTestCaseList (batchResults): 157 caseList = [] 158 caseSet = set() 159 160 for batchResult in batchResults: 161 for result in batchResult.results: 162 if not result.name in caseSet: 163 caseList.append(result.name) 164 caseSet.add(result.name) 165 166 return caseList 167 168def computeUnifiedResults (batchResults): 169 170 def genResultMap (batchResult): 171 resMap = {} 172 for result in batchResult.results: 173 resMap[result.name] = result 174 return resMap 175 176 resultMap = [genResultMap(r) for r in batchResults] 177 caseList = computeUnifiedTestCaseList(batchResults) 178 results = [] 179 180 for caseName in caseList: 181 statusCodes = [] 182 183 for i in range(0, len(batchResults)): 184 result = resultMap[i][caseName] if caseName in resultMap[i] else None 185 statusCode = result.statusCode if result != None else 'Missing' 186 statusCodes.append(statusCode) 187 188 results.append(MultiResult(caseName, statusCodes)) 189 190 return results 191 192def allStatusCodesEqual (result): 193 firstCode = result.statusCodes[0] 194 for i in range(1, len(result.statusCodes)): 195 if result.statusCodes[i] != firstCode: 196 return False 197 return True 198 199def computeDiffResults (unifiedResults): 200 diff = [] 201 for result in unifiedResults: 202 if not allStatusCodesEqual(result): 203 diff.append(result) 204 return diff 205 206def genCompareReport (batchResults, title, ignoreCases): 207 class TableRow: 208 def __init__ (self, testCaseName, innerHTML): 209 self.testCaseName = testCaseName 210 self.innerHTML = innerHTML 211 212 unifiedResults = computeUnifiedResults(batchResults) 213 diffResults = computeDiffResults(unifiedResults) 214 rows = [] 215 216 # header 217 headerCol = '<td class="Header">Test case</td>\n' 218 for batchResult in batchResults: 219 headerCol += '<td class="Header">%s</td>\n' % batchResult.name 220 rows.append(TableRow(None, headerCol)) 221 222 # results 223 for result in diffResults: 224 col = '<td class="CasePath">%s</td>\n' % result.name 225 for statusCode in result.statusCodes: 226 col += '<td class="%s">%s</td>\n' % (statusCode, statusCode) 227 228 rows.append(TableRow(result.name, col)) 229 230 tableStr = "" 231 for row in rows: 232 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases): 233 tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML 234 else: 235 tableStr += '<tr>\n%s</tr>\n' % row.innerHTML 236 237 html = COMPARE_REPORT_TMPL 238 html = html.replace("${TITLE}", title) 239 html = html.replace("${RESULTS}", tableStr) 240 241 return html 242 243def matchesAnyPattern (name, patterns): 244 for pattern in patterns: 245 if fnmatch.fnmatch(name, pattern): 246 return True 247 return False 248 249def statusCodesMatch (refResult, resResult): 250 return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult 251 252def compareBatchResults (referenceBatch, resultBatch, ignoreCases): 253 unifiedResults = computeUnifiedResults([referenceBatch, resultBatch]) 254 failedCases = [] 255 256 for result in unifiedResults: 257 if not matchesAnyPattern(result.name, ignoreCases): 258 refResult = result.statusCodes[0] 259 resResult = result.statusCodes[1] 260 261 if not statusCodesMatch(refResult, resResult): 262 failedCases.append(result) 263 264 return failedCases 265 266def getUnusedPort (): 267 # \note Not 100%-proof method as other apps may grab this port before we launch execserver 268 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 269 s.bind(('localhost', 0)) 270 addr, port = s.getsockname() 271 s.close() 272 return port 273 274def runNightly (config): 275 build(config.buildConfig, config.generator) 276 277 # Run parameters 278 timestamp = getResultTimestamp() 279 logDir = os.path.join(BASE_LOGS_DIR, config.name) 280 testLogPath = os.path.join(logDir, timestamp + ".qpa") 281 infoLogPath = os.path.join(logDir, timestamp + ".txt") 282 csvLogPath = os.path.join(logDir, timestamp + ".csv") 283 compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv") 284 port = getUnusedPort() 285 286 if not os.path.exists(logDir): 287 os.makedirs(logDir) 288 289 if os.path.exists(testLogPath) or os.path.exists(infoLogPath): 290 raise Exception("Result '%s' already exists", timestamp) 291 292 # Paths, etc. 293 binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName)) 294 workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName)) 295 296 execArgs = [ 297 config.getBinaryPath(EXECUTOR_PATH), 298 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH), 299 '--port=%d' % port, 300 '--binaryname=%s' % binaryName, 301 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "), 302 '--workdir=%s' % workingDir, 303 '--caselistdir=%s' % CASELIST_PATH, 304 '--testset=%s' % string.join(config.testset, ","), 305 '--out=%s' % testLogPath, 306 '--info=%s' % infoLogPath, 307 '--summary=no' 308 ] 309 310 if len(config.exclude) > 0: 311 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")] 312 313 execute(execArgs) 314 315 # Translate to CSV for comparison purposes 316 lastResultCsv = parseAsCSV(testLogPath, config) 317 writeFile(csvLogPath, lastResultCsv) 318 319 if os.path.exists(compareLogPath): 320 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference") 321 else: 322 refBatchResult = None 323 324 # Generate comparison report 325 compareFilenames = getCompareFilenames(logDir) 326 batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames] 327 328 if refBatchResult != None: 329 batchResults = [refBatchResult] + batchResults 330 331 writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore)) 332 print("Comparison report written to %s" % COMPARE_REPORT_NAME) 333 334 # Compare to reference 335 if refBatchResult != None: 336 curBatchResult = BatchResult("current") 337 curBatchResult.results = parseResultCsv(lastResultCsv) 338 failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore) 339 340 print("") 341 for result in failedCases: 342 print("MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])) 343 344 print("") 345 print("%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")) 346 347 if len(failedCases) > 0: 348 return False 349 350 return True 351 352# Configurations 353 354DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR 355DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR 356 357WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl']) 358ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu']) 359 360BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable'] 361 362CONFIGS = [ 363 NightlyRunConfig( 364 name = "wgl_x64_release_gles2", 365 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 366 generator = DEFAULT_WIN64_GENERATOR, 367 binaryName = "modules/gles2/deqp-gles2", 368 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 369 testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"], 370 exclude = [ 371 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*", 372 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*", 373 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*", 374 ], 375 ignore = [] 376 ), 377 NightlyRunConfig( 378 name = "wgl_x64_release_gles3", 379 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 380 generator = DEFAULT_WIN64_GENERATOR, 381 binaryName = "modules/gles3/deqp-gles3", 382 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 383 testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"], 384 exclude = [ 385 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*", 386 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*", 387 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*", 388 ], 389 ignore = [ 390 "dEQP-GLES3.functional.transform_feedback.*", 391 "dEQP-GLES3.functional.occlusion_query.*", 392 "dEQP-GLES3.functional.lifetime.*", 393 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops", 394 ] 395 ), 396 NightlyRunConfig( 397 name = "wgl_x64_release_gles31", 398 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 399 generator = DEFAULT_WIN64_GENERATOR, 400 binaryName = "modules/gles31/deqp-gles31", 401 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 402 testset = ["dEQP-GLES31.*"], 403 exclude = [], 404 ignore = [ 405 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3", 406 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer", 407 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset", 408 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint", 409 "dEQP-GLES31.functional.blend_equation_advanced.basic.*", 410 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*", 411 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*", 412 "dEQP-GLES31.functional.uniform_location.*", 413 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv", 414 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv", 415 "dEQP-GLES31.functional.debug.error_filters.case_0", 416 "dEQP-GLES31.functional.debug.error_filters.case_2", 417 ] 418 ), 419 NightlyRunConfig( 420 name = "wgl_x64_release_gl3", 421 buildConfig = WGL_X64_RELEASE_BUILD_CFG, 422 generator = DEFAULT_WIN64_GENERATOR, 423 binaryName = "modules/gl3/deqp-gl3", 424 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 425 testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"], 426 exclude = [ 427 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*", 428 "dEQP-GL3.functional.shaders.loops.*while*only_continue*", 429 "dEQP-GL3.functional.shaders.loops.*while*double_continue*", 430 ], 431 ignore = [ 432 "dEQP-GL3.functional.transform_feedback.*" 433 ] 434 ), 435 NightlyRunConfig( 436 name = "arm_gles3_emu_x32_egl", 437 buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG, 438 generator = DEFAULT_WIN32_GENERATOR, 439 binaryName = "modules/egl/deqp-egl", 440 args = BASE_ARGS, 441 testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"], 442 exclude = [ 443 "dEQP-EGL.functional.sharing.gles2.multithread.*", 444 "dEQP-EGL.functional.multithread.*", 445 ], 446 ignore = [] 447 ), 448 NightlyRunConfig( 449 name = "opencl_x64_release", 450 buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']), 451 generator = DEFAULT_WIN64_GENERATOR, 452 binaryName = "modules/opencl/deqp-opencl", 453 args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS, 454 testset = ["dEQP-CL.*"], 455 exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"], 456 ignore = [ 457 "dEQP-CL.scheduler.random.*", 458 "dEQP-CL.language.set_kernel_arg.random_structs.*", 459 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset", 460 "dEQP-CL.language.call_function.arguments.random_structs.*", 461 "dEQP-CL.language.call_kernel.random_structs.*", 462 "dEQP-CL.language.inf_nan.nan.frexp.float", 463 "dEQP-CL.language.inf_nan.nan.lgamma_r.float", 464 "dEQP-CL.language.inf_nan.nan.modf.float", 465 "dEQP-CL.language.inf_nan.nan.sqrt.float", 466 "dEQP-CL.api.multithread.*", 467 "dEQP-CL.api.callback.random.nested.*", 468 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb", 469 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb", 470 "dEQP-CL.image.addressing_filtering12.1d_array.*", 471 "dEQP-CL.image.addressing_filtering12.2d_array.*" 472 ] 473 ) 474] 475 476if __name__ == "__main__": 477 config = None 478 479 if len(sys.argv) == 2: 480 cfgName = sys.argv[1] 481 for curCfg in CONFIGS: 482 if curCfg.name == cfgName: 483 config = curCfg 484 break 485 486 if config != None: 487 isOk = runNightly(config) 488 if not isOk: 489 sys.exit(-1) 490 else: 491 print("%s: [config]" % sys.argv[0]) 492 print("") 493 print(" Available configs:") 494 for config in CONFIGS: 495 print(" %s" % config.name) 496 sys.exit(-1) 497