1*35238bceSAndroid Build Coastguard Worker# -*- coding: utf-8 -*- 2*35238bceSAndroid Build Coastguard Worker 3*35238bceSAndroid Build Coastguard Worker#------------------------------------------------------------------------- 4*35238bceSAndroid Build Coastguard Worker# drawElements Quality Program utilities 5*35238bceSAndroid Build Coastguard Worker# -------------------------------------- 6*35238bceSAndroid Build Coastguard Worker# 7*35238bceSAndroid Build Coastguard Worker# Copyright 2015 The Android Open Source Project 8*35238bceSAndroid Build Coastguard Worker# 9*35238bceSAndroid Build Coastguard Worker# Licensed under the Apache License, Version 2.0 (the "License"); 10*35238bceSAndroid Build Coastguard Worker# you may not use this file except in compliance with the License. 11*35238bceSAndroid Build Coastguard Worker# You may obtain a copy of the License at 12*35238bceSAndroid Build Coastguard Worker# 13*35238bceSAndroid Build Coastguard Worker# http://www.apache.org/licenses/LICENSE-2.0 14*35238bceSAndroid Build Coastguard Worker# 15*35238bceSAndroid Build Coastguard Worker# Unless required by applicable law or agreed to in writing, software 16*35238bceSAndroid Build Coastguard Worker# distributed under the License is distributed on an "AS IS" BASIS, 17*35238bceSAndroid Build Coastguard Worker# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18*35238bceSAndroid Build Coastguard Worker# See the License for the specific language governing permissions and 19*35238bceSAndroid Build Coastguard Worker# limitations under the License. 20*35238bceSAndroid Build Coastguard Worker# 21*35238bceSAndroid Build Coastguard Worker#------------------------------------------------------------------------- 22*35238bceSAndroid Build Coastguard Worker 23*35238bceSAndroid Build Coastguard Workerfrom ctsbuild.common import * 24*35238bceSAndroid Build Coastguard Workerfrom ctsbuild.config import * 25*35238bceSAndroid Build Coastguard Workerfrom ctsbuild.build import * 26*35238bceSAndroid Build Coastguard Worker 27*35238bceSAndroid Build Coastguard Workerimport os 28*35238bceSAndroid Build Coastguard Workerimport sys 29*35238bceSAndroid Build Coastguard Workerimport string 30*35238bceSAndroid Build Coastguard Workerimport socket 31*35238bceSAndroid Build Coastguard Workerimport fnmatch 32*35238bceSAndroid Build Coastguard Workerfrom datetime import datetime 33*35238bceSAndroid Build Coastguard Worker 34*35238bceSAndroid Build Coastguard WorkerBASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly")) 35*35238bceSAndroid Build Coastguard WorkerBASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build") 36*35238bceSAndroid Build Coastguard WorkerBASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs") 37*35238bceSAndroid Build Coastguard WorkerBASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs") 38*35238bceSAndroid Build Coastguard Worker 39*35238bceSAndroid Build Coastguard WorkerEXECUTOR_PATH = "executor/executor" 40*35238bceSAndroid Build Coastguard WorkerLOG_TO_CSV_PATH = "executor/testlog-to-csv" 41*35238bceSAndroid Build Coastguard WorkerEXECSERVER_PATH = "execserver/execserver" 42*35238bceSAndroid Build Coastguard Worker 43*35238bceSAndroid Build Coastguard WorkerCASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data") 44*35238bceSAndroid Build Coastguard Worker 45*35238bceSAndroid Build Coastguard WorkerCOMPARE_NUM_RESULTS = 4 46*35238bceSAndroid Build Coastguard WorkerCOMPARE_REPORT_NAME = "nightly-report.html" 47*35238bceSAndroid Build Coastguard Worker 48*35238bceSAndroid Build Coastguard WorkerCOMPARE_REPORT_TMPL = ''' 49*35238bceSAndroid Build Coastguard Worker<html> 50*35238bceSAndroid Build Coastguard Worker<head> 51*35238bceSAndroid Build Coastguard Worker<title>${TITLE}</title> 52*35238bceSAndroid Build Coastguard Worker<style type="text/css"> 53*35238bceSAndroid Build Coastguard Worker<!-- 54*35238bceSAndroid Build Coastguard Workerbody { font: serif; font-size: 1em; } 55*35238bceSAndroid Build Coastguard Workertable { border-spacing: 0; border-collapse: collapse; } 56*35238bceSAndroid Build Coastguard Workertd { border-width: 1px; border-style: solid; border-color: #808080; } 57*35238bceSAndroid Build Coastguard Worker.Header { font-weight: bold; font-size: 1em; border-style: none; } 58*35238bceSAndroid Build Coastguard Worker.CasePath { } 59*35238bceSAndroid Build Coastguard Worker.Pass { background: #80ff80; } 60*35238bceSAndroid Build Coastguard Worker.Fail { background: #ff4040; } 61*35238bceSAndroid Build Coastguard Worker.QualityWarning { background: #ffff00; } 62*35238bceSAndroid Build Coastguard Worker.CompabilityWarning { background: #ffff00; } 63*35238bceSAndroid Build Coastguard Worker.Pending { background: #808080; } 64*35238bceSAndroid Build Coastguard Worker.Running { background: #d3d3d3; } 65*35238bceSAndroid Build Coastguard Worker.NotSupported { background: #ff69b4; } 66*35238bceSAndroid Build Coastguard Worker.ResourceError { background: #ff4040; } 67*35238bceSAndroid Build Coastguard Worker.InternalError { background: #ff1493; } 68*35238bceSAndroid Build Coastguard Worker.Canceled { background: #808080; } 69*35238bceSAndroid Build Coastguard Worker.Crash { background: #ffa500; } 70*35238bceSAndroid Build Coastguard Worker.Timeout { background: #ffa500; } 71*35238bceSAndroid Build Coastguard Worker.Disabled { background: #808080; } 72*35238bceSAndroid Build Coastguard Worker.Missing { background: #808080; } 73*35238bceSAndroid Build Coastguard Worker.Ignored { opacity: 0.5; } 74*35238bceSAndroid Build Coastguard Worker--> 75*35238bceSAndroid Build Coastguard Worker</style> 76*35238bceSAndroid Build Coastguard Worker</head> 77*35238bceSAndroid Build Coastguard Worker<body> 78*35238bceSAndroid Build Coastguard Worker<h1>${TITLE}</h1> 79*35238bceSAndroid Build Coastguard Worker<table> 80*35238bceSAndroid Build Coastguard Worker${RESULTS} 81*35238bceSAndroid Build Coastguard Worker</table> 82*35238bceSAndroid Build Coastguard Worker</body> 83*35238bceSAndroid Build Coastguard Worker</html> 84*35238bceSAndroid Build Coastguard Worker''' 85*35238bceSAndroid Build Coastguard Worker 86*35238bceSAndroid Build Coastguard Workerclass NightlyRunConfig: 87*35238bceSAndroid Build Coastguard Worker def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []): 88*35238bceSAndroid Build Coastguard Worker self.name = name 89*35238bceSAndroid Build Coastguard Worker self.buildConfig = buildConfig 90*35238bceSAndroid Build Coastguard Worker self.generator = generator 91*35238bceSAndroid Build Coastguard Worker self.binaryName = binaryName 92*35238bceSAndroid Build Coastguard Worker self.testset = testset 93*35238bceSAndroid Build Coastguard Worker self.args = args 94*35238bceSAndroid Build Coastguard Worker self.exclude = exclude 95*35238bceSAndroid Build Coastguard Worker self.ignore = ignore 96*35238bceSAndroid Build Coastguard Worker 97*35238bceSAndroid Build Coastguard Worker def getBinaryPath(self, basePath): 98*35238bceSAndroid Build Coastguard Worker return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath)) 99*35238bceSAndroid Build Coastguard Worker 100*35238bceSAndroid Build Coastguard Workerclass NightlyBuildConfig(BuildConfig): 101*35238bceSAndroid Build Coastguard Worker def __init__(self, name, buildType, args): 102*35238bceSAndroid Build Coastguard Worker BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args) 103*35238bceSAndroid Build Coastguard Worker 104*35238bceSAndroid Build Coastguard Workerclass TestCaseResult: 105*35238bceSAndroid Build Coastguard Worker def __init__ (self, name, statusCode): 106*35238bceSAndroid Build Coastguard Worker self.name = name 107*35238bceSAndroid Build Coastguard Worker self.statusCode = statusCode 108*35238bceSAndroid Build Coastguard Worker 109*35238bceSAndroid Build Coastguard Workerclass MultiResult: 110*35238bceSAndroid Build Coastguard Worker def __init__ (self, name, statusCodes): 111*35238bceSAndroid Build Coastguard Worker self.name = name 112*35238bceSAndroid Build Coastguard Worker self.statusCodes = statusCodes 113*35238bceSAndroid Build Coastguard Worker 114*35238bceSAndroid Build Coastguard Workerclass BatchResult: 115*35238bceSAndroid Build Coastguard Worker def __init__ (self, name): 116*35238bceSAndroid Build Coastguard Worker self.name = name 117*35238bceSAndroid Build Coastguard Worker self.results = [] 118*35238bceSAndroid Build Coastguard Worker 119*35238bceSAndroid Build Coastguard Workerdef parseResultCsv (data): 120*35238bceSAndroid Build Coastguard Worker lines = data.splitlines()[1:] 121*35238bceSAndroid Build Coastguard Worker results = [] 122*35238bceSAndroid Build Coastguard Worker 123*35238bceSAndroid Build Coastguard Worker for line in lines: 124*35238bceSAndroid Build Coastguard Worker items = line.split(",") 125*35238bceSAndroid Build Coastguard Worker results.append(TestCaseResult(items[0], items[1])) 126*35238bceSAndroid Build Coastguard Worker 127*35238bceSAndroid Build Coastguard Worker return results 128*35238bceSAndroid Build Coastguard Worker 129*35238bceSAndroid Build Coastguard Workerdef readTestCaseResultsFromCSV (filename): 130*35238bceSAndroid Build Coastguard Worker return parseResultCsv(readFile(filename)) 131*35238bceSAndroid Build Coastguard Worker 132*35238bceSAndroid Build Coastguard Workerdef readBatchResultFromCSV (filename, batchResultName = None): 133*35238bceSAndroid Build Coastguard Worker batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename)) 134*35238bceSAndroid Build Coastguard Worker batchResult.results = readTestCaseResultsFromCSV(filename) 135*35238bceSAndroid Build Coastguard Worker return batchResult 136*35238bceSAndroid Build Coastguard Worker 137*35238bceSAndroid Build Coastguard Workerdef getResultTimestamp (): 138*35238bceSAndroid Build Coastguard Worker return datetime.now().strftime("%Y-%m-%d-%H-%M") 139*35238bceSAndroid Build Coastguard Worker 140*35238bceSAndroid Build Coastguard Workerdef getCompareFilenames (logsDir): 141*35238bceSAndroid Build Coastguard Worker files = [] 142*35238bceSAndroid Build Coastguard Worker for file in os.listdir(logsDir): 143*35238bceSAndroid Build Coastguard Worker fullPath = os.path.join(logsDir, file) 144*35238bceSAndroid Build Coastguard Worker if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"): 145*35238bceSAndroid Build Coastguard Worker files.append(fullPath) 146*35238bceSAndroid Build Coastguard Worker files.sort() 147*35238bceSAndroid Build Coastguard Worker 148*35238bceSAndroid Build Coastguard Worker return files[-COMPARE_NUM_RESULTS:] 149*35238bceSAndroid Build Coastguard Worker 150*35238bceSAndroid Build Coastguard Workerdef parseAsCSV (logPath, config): 151*35238bceSAndroid Build Coastguard Worker args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath] 152*35238bceSAndroid Build Coastguard Worker proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 153*35238bceSAndroid Build Coastguard Worker out, err = proc.communicate() 154*35238bceSAndroid Build Coastguard Worker return out 155*35238bceSAndroid Build Coastguard Worker 156*35238bceSAndroid Build Coastguard Workerdef computeUnifiedTestCaseList (batchResults): 157*35238bceSAndroid Build Coastguard Worker caseList = [] 158*35238bceSAndroid Build Coastguard Worker caseSet = set() 159*35238bceSAndroid Build Coastguard Worker 160*35238bceSAndroid Build Coastguard Worker for batchResult in batchResults: 161*35238bceSAndroid Build Coastguard Worker for result in batchResult.results: 162*35238bceSAndroid Build Coastguard Worker if not result.name in caseSet: 163*35238bceSAndroid Build Coastguard Worker caseList.append(result.name) 164*35238bceSAndroid Build Coastguard Worker caseSet.add(result.name) 165*35238bceSAndroid Build Coastguard Worker 166*35238bceSAndroid Build Coastguard Worker return caseList 167*35238bceSAndroid Build Coastguard Worker 168*35238bceSAndroid Build Coastguard Workerdef computeUnifiedResults (batchResults): 169*35238bceSAndroid Build Coastguard Worker 170*35238bceSAndroid Build Coastguard Worker def genResultMap (batchResult): 171*35238bceSAndroid Build Coastguard Worker resMap = {} 172*35238bceSAndroid Build Coastguard Worker for result in batchResult.results: 173*35238bceSAndroid Build Coastguard Worker resMap[result.name] = result 174*35238bceSAndroid Build Coastguard Worker return resMap 175*35238bceSAndroid Build Coastguard Worker 176*35238bceSAndroid Build Coastguard Worker resultMap = [genResultMap(r) for r in batchResults] 177*35238bceSAndroid Build Coastguard Worker caseList = computeUnifiedTestCaseList(batchResults) 178*35238bceSAndroid Build Coastguard Worker results = [] 179*35238bceSAndroid Build Coastguard Worker 180*35238bceSAndroid Build Coastguard Worker for caseName in caseList: 181*35238bceSAndroid Build Coastguard Worker statusCodes = [] 182*35238bceSAndroid Build Coastguard Worker 183*35238bceSAndroid Build Coastguard Worker for i in range(0, len(batchResults)): 184*35238bceSAndroid Build Coastguard Worker result = resultMap[i][caseName] if caseName in resultMap[i] else None 185*35238bceSAndroid Build Coastguard Worker statusCode = result.statusCode if result != None else 'Missing' 186*35238bceSAndroid Build Coastguard Worker statusCodes.append(statusCode) 187*35238bceSAndroid Build Coastguard Worker 188*35238bceSAndroid Build Coastguard Worker results.append(MultiResult(caseName, statusCodes)) 189*35238bceSAndroid Build Coastguard Worker 190*35238bceSAndroid Build Coastguard Worker return results 191*35238bceSAndroid Build Coastguard Worker 192*35238bceSAndroid Build Coastguard Workerdef allStatusCodesEqual (result): 193*35238bceSAndroid Build Coastguard Worker firstCode = result.statusCodes[0] 194*35238bceSAndroid Build Coastguard Worker for i in range(1, len(result.statusCodes)): 195*35238bceSAndroid Build Coastguard Worker if result.statusCodes[i] != firstCode: 196*35238bceSAndroid Build Coastguard Worker return False 197*35238bceSAndroid Build Coastguard Worker return True 198*35238bceSAndroid Build Coastguard Worker 199*35238bceSAndroid Build Coastguard Workerdef computeDiffResults (unifiedResults): 200*35238bceSAndroid Build Coastguard Worker diff = [] 201*35238bceSAndroid Build Coastguard Worker for result in unifiedResults: 202*35238bceSAndroid Build Coastguard Worker if not allStatusCodesEqual(result): 203*35238bceSAndroid Build Coastguard Worker diff.append(result) 204*35238bceSAndroid Build Coastguard Worker return diff 205*35238bceSAndroid Build Coastguard Worker 206*35238bceSAndroid Build Coastguard Workerdef genCompareReport (batchResults, title, ignoreCases): 207*35238bceSAndroid Build Coastguard Worker class TableRow: 208*35238bceSAndroid Build Coastguard Worker def __init__ (self, testCaseName, innerHTML): 209*35238bceSAndroid Build Coastguard Worker self.testCaseName = testCaseName 210*35238bceSAndroid Build Coastguard Worker self.innerHTML = innerHTML 211*35238bceSAndroid Build Coastguard Worker 212*35238bceSAndroid Build Coastguard Worker unifiedResults = computeUnifiedResults(batchResults) 213*35238bceSAndroid Build Coastguard Worker diffResults = computeDiffResults(unifiedResults) 214*35238bceSAndroid Build Coastguard Worker rows = [] 215*35238bceSAndroid Build Coastguard Worker 216*35238bceSAndroid Build Coastguard Worker # header 217*35238bceSAndroid Build Coastguard Worker headerCol = '<td class="Header">Test case</td>\n' 218*35238bceSAndroid Build Coastguard Worker for batchResult in batchResults: 219*35238bceSAndroid Build Coastguard Worker headerCol += '<td class="Header">%s</td>\n' % batchResult.name 220*35238bceSAndroid Build Coastguard Worker rows.append(TableRow(None, headerCol)) 221*35238bceSAndroid Build Coastguard Worker 222*35238bceSAndroid Build Coastguard Worker # results 223*35238bceSAndroid Build Coastguard Worker for result in diffResults: 224*35238bceSAndroid Build Coastguard Worker col = '<td class="CasePath">%s</td>\n' % result.name 225*35238bceSAndroid Build Coastguard Worker for statusCode in result.statusCodes: 226*35238bceSAndroid Build Coastguard Worker col += '<td class="%s">%s</td>\n' % (statusCode, statusCode) 227*35238bceSAndroid Build Coastguard Worker 228*35238bceSAndroid Build Coastguard Worker rows.append(TableRow(result.name, col)) 229*35238bceSAndroid Build Coastguard Worker 230*35238bceSAndroid Build Coastguard Worker tableStr = "" 231*35238bceSAndroid Build Coastguard Worker for row in rows: 232*35238bceSAndroid Build Coastguard Worker if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases): 233*35238bceSAndroid Build Coastguard Worker tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML 234*35238bceSAndroid Build Coastguard Worker else: 235*35238bceSAndroid Build Coastguard Worker tableStr += '<tr>\n%s</tr>\n' % row.innerHTML 236*35238bceSAndroid Build Coastguard Worker 237*35238bceSAndroid Build Coastguard Worker html = COMPARE_REPORT_TMPL 238*35238bceSAndroid Build Coastguard Worker html = html.replace("${TITLE}", title) 239*35238bceSAndroid Build Coastguard Worker html = html.replace("${RESULTS}", tableStr) 240*35238bceSAndroid Build Coastguard Worker 241*35238bceSAndroid Build Coastguard Worker return html 242*35238bceSAndroid Build Coastguard Worker 243*35238bceSAndroid Build Coastguard Workerdef matchesAnyPattern (name, patterns): 244*35238bceSAndroid Build Coastguard Worker for pattern in patterns: 245*35238bceSAndroid Build Coastguard Worker if fnmatch.fnmatch(name, pattern): 246*35238bceSAndroid Build Coastguard Worker return True 247*35238bceSAndroid Build Coastguard Worker return False 248*35238bceSAndroid Build Coastguard Worker 249*35238bceSAndroid Build Coastguard Workerdef statusCodesMatch (refResult, resResult): 250*35238bceSAndroid Build Coastguard Worker return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult 251*35238bceSAndroid Build Coastguard Worker 252*35238bceSAndroid Build Coastguard Workerdef compareBatchResults (referenceBatch, resultBatch, ignoreCases): 253*35238bceSAndroid Build Coastguard Worker unifiedResults = computeUnifiedResults([referenceBatch, resultBatch]) 254*35238bceSAndroid Build Coastguard Worker failedCases = [] 255*35238bceSAndroid Build Coastguard Worker 256*35238bceSAndroid Build Coastguard Worker for result in unifiedResults: 257*35238bceSAndroid Build Coastguard Worker if not matchesAnyPattern(result.name, ignoreCases): 258*35238bceSAndroid Build Coastguard Worker refResult = result.statusCodes[0] 259*35238bceSAndroid Build Coastguard Worker resResult = result.statusCodes[1] 260*35238bceSAndroid Build Coastguard Worker 261*35238bceSAndroid Build Coastguard Worker if not statusCodesMatch(refResult, resResult): 262*35238bceSAndroid Build Coastguard Worker failedCases.append(result) 263*35238bceSAndroid Build Coastguard Worker 264*35238bceSAndroid Build Coastguard Worker return failedCases 265*35238bceSAndroid Build Coastguard Worker 266*35238bceSAndroid Build Coastguard Workerdef getUnusedPort (): 267*35238bceSAndroid Build Coastguard Worker # \note Not 100%-proof method as other apps may grab this port before we launch execserver 268*35238bceSAndroid Build Coastguard Worker s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 269*35238bceSAndroid Build Coastguard Worker s.bind(('localhost', 0)) 270*35238bceSAndroid Build Coastguard Worker addr, port = s.getsockname() 271*35238bceSAndroid Build Coastguard Worker s.close() 272*35238bceSAndroid Build Coastguard Worker return port 273*35238bceSAndroid Build Coastguard Worker 274*35238bceSAndroid Build Coastguard Workerdef runNightly (config): 275*35238bceSAndroid Build Coastguard Worker build(config.buildConfig, config.generator) 276*35238bceSAndroid Build Coastguard Worker 277*35238bceSAndroid Build Coastguard Worker # Run parameters 278*35238bceSAndroid Build Coastguard Worker timestamp = getResultTimestamp() 279*35238bceSAndroid Build Coastguard Worker logDir = os.path.join(BASE_LOGS_DIR, config.name) 280*35238bceSAndroid Build Coastguard Worker testLogPath = os.path.join(logDir, timestamp + ".qpa") 281*35238bceSAndroid Build Coastguard Worker infoLogPath = os.path.join(logDir, timestamp + ".txt") 282*35238bceSAndroid Build Coastguard Worker csvLogPath = os.path.join(logDir, timestamp + ".csv") 283*35238bceSAndroid Build Coastguard Worker compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv") 284*35238bceSAndroid Build Coastguard Worker port = getUnusedPort() 285*35238bceSAndroid Build Coastguard Worker 286*35238bceSAndroid Build Coastguard Worker if not os.path.exists(logDir): 287*35238bceSAndroid Build Coastguard Worker os.makedirs(logDir) 288*35238bceSAndroid Build Coastguard Worker 289*35238bceSAndroid Build Coastguard Worker if os.path.exists(testLogPath) or os.path.exists(infoLogPath): 290*35238bceSAndroid Build Coastguard Worker raise Exception("Result '%s' already exists", timestamp) 291*35238bceSAndroid Build Coastguard Worker 292*35238bceSAndroid Build Coastguard Worker # Paths, etc. 293*35238bceSAndroid Build Coastguard Worker binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName)) 294*35238bceSAndroid Build Coastguard Worker workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName)) 295*35238bceSAndroid Build Coastguard Worker 296*35238bceSAndroid Build Coastguard Worker execArgs = [ 297*35238bceSAndroid Build Coastguard Worker config.getBinaryPath(EXECUTOR_PATH), 298*35238bceSAndroid Build Coastguard Worker '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH), 299*35238bceSAndroid Build Coastguard Worker '--port=%d' % port, 300*35238bceSAndroid Build Coastguard Worker '--binaryname=%s' % binaryName, 301*35238bceSAndroid Build Coastguard Worker '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "), 302*35238bceSAndroid Build Coastguard Worker '--workdir=%s' % workingDir, 303*35238bceSAndroid Build Coastguard Worker '--caselistdir=%s' % CASELIST_PATH, 304*35238bceSAndroid Build Coastguard Worker '--testset=%s' % string.join(config.testset, ","), 305*35238bceSAndroid Build Coastguard Worker '--out=%s' % testLogPath, 306*35238bceSAndroid Build Coastguard Worker '--info=%s' % infoLogPath, 307*35238bceSAndroid Build Coastguard Worker '--summary=no' 308*35238bceSAndroid Build Coastguard Worker ] 309*35238bceSAndroid Build Coastguard Worker 310*35238bceSAndroid Build Coastguard Worker if len(config.exclude) > 0: 311*35238bceSAndroid Build Coastguard Worker execArgs += ['--exclude=%s' % string.join(config.exclude, ",")] 312*35238bceSAndroid Build Coastguard Worker 313*35238bceSAndroid Build Coastguard Worker execute(execArgs) 314*35238bceSAndroid Build Coastguard Worker 315*35238bceSAndroid Build Coastguard Worker # Translate to CSV for comparison purposes 316*35238bceSAndroid Build Coastguard Worker lastResultCsv = parseAsCSV(testLogPath, config) 317*35238bceSAndroid Build Coastguard Worker writeFile(csvLogPath, lastResultCsv) 318*35238bceSAndroid Build Coastguard Worker 319*35238bceSAndroid Build Coastguard Worker if os.path.exists(compareLogPath): 320*35238bceSAndroid Build Coastguard Worker refBatchResult = readBatchResultFromCSV(compareLogPath, "reference") 321*35238bceSAndroid Build Coastguard Worker else: 322*35238bceSAndroid Build Coastguard Worker refBatchResult = None 323*35238bceSAndroid Build Coastguard Worker 324*35238bceSAndroid Build Coastguard Worker # Generate comparison report 325*35238bceSAndroid Build Coastguard Worker compareFilenames = getCompareFilenames(logDir) 326*35238bceSAndroid Build Coastguard Worker batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames] 327*35238bceSAndroid Build Coastguard Worker 328*35238bceSAndroid Build Coastguard Worker if refBatchResult != None: 329*35238bceSAndroid Build Coastguard Worker batchResults = [refBatchResult] + batchResults 330*35238bceSAndroid Build Coastguard Worker 331*35238bceSAndroid Build Coastguard Worker writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore)) 332*35238bceSAndroid Build Coastguard Worker print("Comparison report written to %s" % COMPARE_REPORT_NAME) 333*35238bceSAndroid Build Coastguard Worker 334*35238bceSAndroid Build Coastguard Worker # Compare to reference 335*35238bceSAndroid Build Coastguard Worker if refBatchResult != None: 336*35238bceSAndroid Build Coastguard Worker curBatchResult = BatchResult("current") 337*35238bceSAndroid Build Coastguard Worker curBatchResult.results = parseResultCsv(lastResultCsv) 338*35238bceSAndroid Build Coastguard Worker failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore) 339*35238bceSAndroid Build Coastguard Worker 340*35238bceSAndroid Build Coastguard Worker print("") 341*35238bceSAndroid Build Coastguard Worker for result in failedCases: 342*35238bceSAndroid Build Coastguard Worker print("MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])) 343*35238bceSAndroid Build Coastguard Worker 344*35238bceSAndroid Build Coastguard Worker print("") 345*35238bceSAndroid Build Coastguard Worker print("%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")) 346*35238bceSAndroid Build Coastguard Worker 347*35238bceSAndroid Build Coastguard Worker if len(failedCases) > 0: 348*35238bceSAndroid Build Coastguard Worker return False 349*35238bceSAndroid Build Coastguard Worker 350*35238bceSAndroid Build Coastguard Worker return True 351*35238bceSAndroid Build Coastguard Worker 352*35238bceSAndroid Build Coastguard Worker# Configurations 353*35238bceSAndroid Build Coastguard Worker 354*35238bceSAndroid Build Coastguard WorkerDEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR 355*35238bceSAndroid Build Coastguard WorkerDEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR 356*35238bceSAndroid Build Coastguard Worker 357*35238bceSAndroid Build Coastguard WorkerWGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl']) 358*35238bceSAndroid Build Coastguard WorkerARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu']) 359*35238bceSAndroid Build Coastguard Worker 360*35238bceSAndroid Build Coastguard WorkerBASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable'] 361*35238bceSAndroid Build Coastguard Worker 362*35238bceSAndroid Build Coastguard WorkerCONFIGS = [ 363*35238bceSAndroid Build Coastguard Worker NightlyRunConfig( 364*35238bceSAndroid Build Coastguard Worker name = "wgl_x64_release_gles2", 365*35238bceSAndroid Build Coastguard Worker buildConfig = WGL_X64_RELEASE_BUILD_CFG, 366*35238bceSAndroid Build Coastguard Worker generator = DEFAULT_WIN64_GENERATOR, 367*35238bceSAndroid Build Coastguard Worker binaryName = "modules/gles2/deqp-gles2", 368*35238bceSAndroid Build Coastguard Worker args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 369*35238bceSAndroid Build Coastguard Worker testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"], 370*35238bceSAndroid Build Coastguard Worker exclude = [ 371*35238bceSAndroid Build Coastguard Worker "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*", 372*35238bceSAndroid Build Coastguard Worker "dEQP-GLES2.functional.shaders.loops.*while*only_continue*", 373*35238bceSAndroid Build Coastguard Worker "dEQP-GLES2.functional.shaders.loops.*while*double_continue*", 374*35238bceSAndroid Build Coastguard Worker ], 375*35238bceSAndroid Build Coastguard Worker ignore = [] 376*35238bceSAndroid Build Coastguard Worker ), 377*35238bceSAndroid Build Coastguard Worker NightlyRunConfig( 378*35238bceSAndroid Build Coastguard Worker name = "wgl_x64_release_gles3", 379*35238bceSAndroid Build Coastguard Worker buildConfig = WGL_X64_RELEASE_BUILD_CFG, 380*35238bceSAndroid Build Coastguard Worker generator = DEFAULT_WIN64_GENERATOR, 381*35238bceSAndroid Build Coastguard Worker binaryName = "modules/gles3/deqp-gles3", 382*35238bceSAndroid Build Coastguard Worker args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 383*35238bceSAndroid Build Coastguard Worker testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"], 384*35238bceSAndroid Build Coastguard Worker exclude = [ 385*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*", 386*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.shaders.loops.*while*only_continue*", 387*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.shaders.loops.*while*double_continue*", 388*35238bceSAndroid Build Coastguard Worker ], 389*35238bceSAndroid Build Coastguard Worker ignore = [ 390*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.transform_feedback.*", 391*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.occlusion_query.*", 392*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.lifetime.*", 393*35238bceSAndroid Build Coastguard Worker "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops", 394*35238bceSAndroid Build Coastguard Worker ] 395*35238bceSAndroid Build Coastguard Worker ), 396*35238bceSAndroid Build Coastguard Worker NightlyRunConfig( 397*35238bceSAndroid Build Coastguard Worker name = "wgl_x64_release_gles31", 398*35238bceSAndroid Build Coastguard Worker buildConfig = WGL_X64_RELEASE_BUILD_CFG, 399*35238bceSAndroid Build Coastguard Worker generator = DEFAULT_WIN64_GENERATOR, 400*35238bceSAndroid Build Coastguard Worker binaryName = "modules/gles31/deqp-gles31", 401*35238bceSAndroid Build Coastguard Worker args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 402*35238bceSAndroid Build Coastguard Worker testset = ["dEQP-GLES31.*"], 403*35238bceSAndroid Build Coastguard Worker exclude = [], 404*35238bceSAndroid Build Coastguard Worker ignore = [ 405*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3", 406*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer", 407*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset", 408*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint", 409*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.blend_equation_advanced.basic.*", 410*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.blend_equation_advanced.srgb.*", 411*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.blend_equation_advanced.barrier.*", 412*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.uniform_location.*", 413*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv", 414*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv", 415*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.debug.error_filters.case_0", 416*35238bceSAndroid Build Coastguard Worker "dEQP-GLES31.functional.debug.error_filters.case_2", 417*35238bceSAndroid Build Coastguard Worker ] 418*35238bceSAndroid Build Coastguard Worker ), 419*35238bceSAndroid Build Coastguard Worker NightlyRunConfig( 420*35238bceSAndroid Build Coastguard Worker name = "wgl_x64_release_gl3", 421*35238bceSAndroid Build Coastguard Worker buildConfig = WGL_X64_RELEASE_BUILD_CFG, 422*35238bceSAndroid Build Coastguard Worker generator = DEFAULT_WIN64_GENERATOR, 423*35238bceSAndroid Build Coastguard Worker binaryName = "modules/gl3/deqp-gl3", 424*35238bceSAndroid Build Coastguard Worker args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 425*35238bceSAndroid Build Coastguard Worker testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"], 426*35238bceSAndroid Build Coastguard Worker exclude = [ 427*35238bceSAndroid Build Coastguard Worker "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*", 428*35238bceSAndroid Build Coastguard Worker "dEQP-GL3.functional.shaders.loops.*while*only_continue*", 429*35238bceSAndroid Build Coastguard Worker "dEQP-GL3.functional.shaders.loops.*while*double_continue*", 430*35238bceSAndroid Build Coastguard Worker ], 431*35238bceSAndroid Build Coastguard Worker ignore = [ 432*35238bceSAndroid Build Coastguard Worker "dEQP-GL3.functional.transform_feedback.*" 433*35238bceSAndroid Build Coastguard Worker ] 434*35238bceSAndroid Build Coastguard Worker ), 435*35238bceSAndroid Build Coastguard Worker NightlyRunConfig( 436*35238bceSAndroid Build Coastguard Worker name = "arm_gles3_emu_x32_egl", 437*35238bceSAndroid Build Coastguard Worker buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG, 438*35238bceSAndroid Build Coastguard Worker generator = DEFAULT_WIN32_GENERATOR, 439*35238bceSAndroid Build Coastguard Worker binaryName = "modules/egl/deqp-egl", 440*35238bceSAndroid Build Coastguard Worker args = BASE_ARGS, 441*35238bceSAndroid Build Coastguard Worker testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"], 442*35238bceSAndroid Build Coastguard Worker exclude = [ 443*35238bceSAndroid Build Coastguard Worker "dEQP-EGL.functional.sharing.gles2.multithread.*", 444*35238bceSAndroid Build Coastguard Worker "dEQP-EGL.functional.multithread.*", 445*35238bceSAndroid Build Coastguard Worker ], 446*35238bceSAndroid Build Coastguard Worker ignore = [] 447*35238bceSAndroid Build Coastguard Worker ), 448*35238bceSAndroid Build Coastguard Worker NightlyRunConfig( 449*35238bceSAndroid Build Coastguard Worker name = "opencl_x64_release", 450*35238bceSAndroid Build Coastguard Worker buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']), 451*35238bceSAndroid Build Coastguard Worker generator = DEFAULT_WIN64_GENERATOR, 452*35238bceSAndroid Build Coastguard Worker binaryName = "modules/opencl/deqp-opencl", 453*35238bceSAndroid Build Coastguard Worker args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS, 454*35238bceSAndroid Build Coastguard Worker testset = ["dEQP-CL.*"], 455*35238bceSAndroid Build Coastguard Worker exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"], 456*35238bceSAndroid Build Coastguard Worker ignore = [ 457*35238bceSAndroid Build Coastguard Worker "dEQP-CL.scheduler.random.*", 458*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.set_kernel_arg.random_structs.*", 459*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset", 460*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.call_function.arguments.random_structs.*", 461*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.call_kernel.random_structs.*", 462*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.inf_nan.nan.frexp.float", 463*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.inf_nan.nan.lgamma_r.float", 464*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.inf_nan.nan.modf.float", 465*35238bceSAndroid Build Coastguard Worker "dEQP-CL.language.inf_nan.nan.sqrt.float", 466*35238bceSAndroid Build Coastguard Worker "dEQP-CL.api.multithread.*", 467*35238bceSAndroid Build Coastguard Worker "dEQP-CL.api.callback.random.nested.*", 468*35238bceSAndroid Build Coastguard Worker "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb", 469*35238bceSAndroid Build Coastguard Worker "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb", 470*35238bceSAndroid Build Coastguard Worker "dEQP-CL.image.addressing_filtering12.1d_array.*", 471*35238bceSAndroid Build Coastguard Worker "dEQP-CL.image.addressing_filtering12.2d_array.*" 472*35238bceSAndroid Build Coastguard Worker ] 473*35238bceSAndroid Build Coastguard Worker ) 474*35238bceSAndroid Build Coastguard Worker] 475*35238bceSAndroid Build Coastguard Worker 476*35238bceSAndroid Build Coastguard Workerif __name__ == "__main__": 477*35238bceSAndroid Build Coastguard Worker config = None 478*35238bceSAndroid Build Coastguard Worker 479*35238bceSAndroid Build Coastguard Worker if len(sys.argv) == 2: 480*35238bceSAndroid Build Coastguard Worker cfgName = sys.argv[1] 481*35238bceSAndroid Build Coastguard Worker for curCfg in CONFIGS: 482*35238bceSAndroid Build Coastguard Worker if curCfg.name == cfgName: 483*35238bceSAndroid Build Coastguard Worker config = curCfg 484*35238bceSAndroid Build Coastguard Worker break 485*35238bceSAndroid Build Coastguard Worker 486*35238bceSAndroid Build Coastguard Worker if config != None: 487*35238bceSAndroid Build Coastguard Worker isOk = runNightly(config) 488*35238bceSAndroid Build Coastguard Worker if not isOk: 489*35238bceSAndroid Build Coastguard Worker sys.exit(-1) 490*35238bceSAndroid Build Coastguard Worker else: 491*35238bceSAndroid Build Coastguard Worker print("%s: [config]" % sys.argv[0]) 492*35238bceSAndroid Build Coastguard Worker print("") 493*35238bceSAndroid Build Coastguard Worker print(" Available configs:") 494*35238bceSAndroid Build Coastguard Worker for config in CONFIGS: 495*35238bceSAndroid Build Coastguard Worker print(" %s" % config.name) 496*35238bceSAndroid Build Coastguard Worker sys.exit(-1) 497