xref: /aosp_15_r20/external/cronet/testing/merge_scripts/standard_gtest_merge_test.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1#!/usr/bin/env vpython3
2# Copyright 2014 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import json
7import logging
8import os
9import shutil
10import six
11import sys
12import tempfile
13import unittest
14
15import common_merge_script_tests
16import six
17
18THIS_DIR = os.path.dirname(os.path.abspath(__file__))
19
20# For 'standard_gtest_merge.py'.
21sys.path.insert(
22    0, os.path.abspath(os.path.join(THIS_DIR, '..', 'resources')))
23
24import mock
25
26import standard_gtest_merge
27
28
29# gtest json output for successfully finished shard #0.
30GOOD_GTEST_JSON_0 = {
31  'all_tests': [
32    'AlignedMemoryTest.DynamicAllocation',
33    'AlignedMemoryTest.ScopedDynamicAllocation',
34    'AlignedMemoryTest.StackAlignment',
35    'AlignedMemoryTest.StaticAlignment',
36  ],
37  'disabled_tests': [
38    'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
39    'FileTest.TouchGetInfo',
40    'MessageLoopTestTypeDefault.EnsureDeletion',
41  ],
42  'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
43  'per_iteration_data': [{
44    'AlignedMemoryTest.DynamicAllocation': [{
45      'elapsed_time_ms': 0,
46      'losless_snippet': True,
47      'output_snippet': 'blah\\n',
48      'output_snippet_base64': 'YmxhaAo=',
49      'status': 'SUCCESS',
50    }],
51    'AlignedMemoryTest.ScopedDynamicAllocation': [{
52      'elapsed_time_ms': 0,
53      'losless_snippet': True,
54      'output_snippet': 'blah\\n',
55      'output_snippet_base64': 'YmxhaAo=',
56      'status': 'SUCCESS',
57    }],
58  }],
59  'test_locations': {
60    'AlignedMemoryTest.DynamicAllocation': {
61      'file': 'foo/bar/allocation_test.cc',
62      'line': 123,
63    },
64    'AlignedMemoryTest.ScopedDynamicAllocation': {
65      'file': 'foo/bar/allocation_test.cc',
66      'line': 456,
67    },
68    # This is a test from a different shard, but this happens in practice and we
69    # should not fail if information is repeated.
70    'AlignedMemoryTest.StaticAlignment': {
71      'file': 'foo/bar/allocation_test.cc',
72      'line': 12,
73    },
74  },
75}
76
77
78# gtest json output for successfully finished shard #1.
79GOOD_GTEST_JSON_1 = {
80  'all_tests': [
81    'AlignedMemoryTest.DynamicAllocation',
82    'AlignedMemoryTest.ScopedDynamicAllocation',
83    'AlignedMemoryTest.StackAlignment',
84    'AlignedMemoryTest.StaticAlignment',
85  ],
86  'disabled_tests': [
87    'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
88    'FileTest.TouchGetInfo',
89    'MessageLoopTestTypeDefault.EnsureDeletion',
90  ],
91  'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
92  'per_iteration_data': [{
93    'AlignedMemoryTest.StackAlignment': [{
94      'elapsed_time_ms': 0,
95      'losless_snippet': True,
96      'output_snippet': 'blah\\n',
97      'output_snippet_base64': 'YmxhaAo=',
98      'status': 'SUCCESS',
99    }],
100    'AlignedMemoryTest.StaticAlignment': [{
101      'elapsed_time_ms': 0,
102      'losless_snippet': True,
103      'output_snippet': 'blah\\n',
104      'output_snippet_base64': 'YmxhaAo=',
105      'status': 'SUCCESS',
106    }],
107  }],
108  'test_locations': {
109    'AlignedMemoryTest.StackAlignment': {
110      'file': 'foo/bar/allocation_test.cc',
111      'line': 789,
112    },
113    'AlignedMemoryTest.StaticAlignment': {
114      'file': 'foo/bar/allocation_test.cc',
115      'line': 12,
116    },
117  },
118}
119
120
121TIMED_OUT_GTEST_JSON_1 = {
122  'disabled_tests': [],
123  'global_tags': [],
124  'all_tests': [
125    'AlignedMemoryTest.DynamicAllocation',
126    'AlignedMemoryTest.ScopedDynamicAllocation',
127    'AlignedMemoryTest.StackAlignment',
128    'AlignedMemoryTest.StaticAlignment',
129  ],
130  'per_iteration_data': [{
131    'AlignedMemoryTest.StackAlignment': [{
132      'elapsed_time_ms': 54000,
133      'losless_snippet': True,
134      'output_snippet': 'timed out',
135      'output_snippet_base64': '',
136      'status': 'FAILURE',
137    }],
138    'AlignedMemoryTest.StaticAlignment': [{
139      'elapsed_time_ms': 0,
140      'losless_snippet': True,
141      'output_snippet': '',
142      'output_snippet_base64': '',
143      'status': 'NOTRUN',
144    }],
145  }],
146  'test_locations': {
147    'AlignedMemoryTest.StackAlignment': {
148      'file': 'foo/bar/allocation_test.cc',
149      'line': 789,
150    },
151    'AlignedMemoryTest.StaticAlignment': {
152      'file': 'foo/bar/allocation_test.cc',
153      'line': 12,
154    },
155  },
156}
157
158# GOOD_GTEST_JSON_0 and GOOD_GTEST_JSON_1 merged.
159GOOD_GTEST_JSON_MERGED = {
160  'all_tests': [
161    'AlignedMemoryTest.DynamicAllocation',
162    'AlignedMemoryTest.ScopedDynamicAllocation',
163    'AlignedMemoryTest.StackAlignment',
164    'AlignedMemoryTest.StaticAlignment',
165  ],
166  'disabled_tests': [
167    'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
168    'FileTest.TouchGetInfo',
169    'MessageLoopTestTypeDefault.EnsureDeletion',
170  ],
171  'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
172  'missing_shards': [],
173  'per_iteration_data': [{
174    'AlignedMemoryTest.DynamicAllocation': [{
175      'elapsed_time_ms': 0,
176      'losless_snippet': True,
177      'output_snippet': 'blah\\n',
178      'output_snippet_base64': 'YmxhaAo=',
179      'status': 'SUCCESS',
180    }],
181    'AlignedMemoryTest.ScopedDynamicAllocation': [{
182      'elapsed_time_ms': 0,
183      'losless_snippet': True,
184      'output_snippet': 'blah\\n',
185      'output_snippet_base64': 'YmxhaAo=',
186      'status': 'SUCCESS',
187    }],
188    'AlignedMemoryTest.StackAlignment': [{
189      'elapsed_time_ms': 0,
190      'losless_snippet': True,
191      'output_snippet': 'blah\\n',
192      'output_snippet_base64': 'YmxhaAo=',
193      'status': 'SUCCESS',
194    }],
195    'AlignedMemoryTest.StaticAlignment': [{
196      'elapsed_time_ms': 0,
197      'losless_snippet': True,
198      'output_snippet': 'blah\\n',
199      'output_snippet_base64': 'YmxhaAo=',
200      'status': 'SUCCESS',
201    }],
202  }],
203  'swarming_summary': {
204    u'shards': [
205      {
206        u'state': u'COMPLETED',
207        u'outputs_ref': {
208          u'view_url': u'blah',
209        },
210      }
211      ],
212  },
213  'test_locations': {
214    'AlignedMemoryTest.StackAlignment': {
215      'file': 'foo/bar/allocation_test.cc',
216      'line': 789,
217    },
218    'AlignedMemoryTest.StaticAlignment': {
219      'file': 'foo/bar/allocation_test.cc',
220      'line': 12,
221    },
222    'AlignedMemoryTest.DynamicAllocation': {
223      'file': 'foo/bar/allocation_test.cc',
224      'line': 123,
225    },
226    'AlignedMemoryTest.ScopedDynamicAllocation': {
227      'file': 'foo/bar/allocation_test.cc',
228      'line': 456,
229    },
230  },
231}
232
233
234# Only shard #1 finished. UNRELIABLE_RESULTS is set.
235BAD_GTEST_JSON_ONLY_1_SHARD = {
236  'all_tests': [
237    'AlignedMemoryTest.DynamicAllocation',
238    'AlignedMemoryTest.ScopedDynamicAllocation',
239    'AlignedMemoryTest.StackAlignment',
240    'AlignedMemoryTest.StaticAlignment',
241  ],
242  'disabled_tests': [
243    'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
244    'FileTest.TouchGetInfo',
245    'MessageLoopTestTypeDefault.EnsureDeletion',
246  ],
247  'global_tags': [
248    'CPU_64_BITS',
249    'MODE_DEBUG',
250    'OS_LINUX',
251    'OS_POSIX',
252    'UNRELIABLE_RESULTS',
253  ],
254  'missing_shards': [0],
255  'per_iteration_data': [{
256    'AlignedMemoryTest.StackAlignment': [{
257      'elapsed_time_ms': 0,
258      'losless_snippet': True,
259      'output_snippet': 'blah\\n',
260      'output_snippet_base64': 'YmxhaAo=',
261      'status': 'SUCCESS',
262    }],
263    'AlignedMemoryTest.StaticAlignment': [{
264      'elapsed_time_ms': 0,
265      'losless_snippet': True,
266      'output_snippet': 'blah\\n',
267      'output_snippet_base64': 'YmxhaAo=',
268      'status': 'SUCCESS',
269    }],
270  }],
271  'test_locations': {
272    'AlignedMemoryTest.StackAlignment': {
273      'file': 'foo/bar/allocation_test.cc',
274      'line': 789,
275    },
276    'AlignedMemoryTest.StaticAlignment': {
277      'file': 'foo/bar/allocation_test.cc',
278      'line': 12,
279    },
280  },
281}
282
283
284# GOOD_GTEST_JSON_0 and TIMED_OUT_GTEST_JSON_1 merged.
285TIMED_OUT_GTEST_JSON_MERGED = {
286  'all_tests': [
287    'AlignedMemoryTest.DynamicAllocation',
288    'AlignedMemoryTest.ScopedDynamicAllocation',
289    'AlignedMemoryTest.StackAlignment',
290    'AlignedMemoryTest.StaticAlignment',
291  ],
292  'disabled_tests': [
293    'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
294    'FileTest.TouchGetInfo',
295    'MessageLoopTestTypeDefault.EnsureDeletion',
296  ],
297  'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
298  'missing_shards': [],
299  'per_iteration_data': [{
300    'AlignedMemoryTest.DynamicAllocation': [{
301      'elapsed_time_ms': 0,
302      'losless_snippet': True,
303      'output_snippet': 'blah\\n',
304      'output_snippet_base64': 'YmxhaAo=',
305      'status': 'SUCCESS',
306    }],
307    'AlignedMemoryTest.ScopedDynamicAllocation': [{
308      'elapsed_time_ms': 0,
309      'losless_snippet': True,
310      'output_snippet': 'blah\\n',
311      'output_snippet_base64': 'YmxhaAo=',
312      'status': 'SUCCESS',
313    }],
314    'AlignedMemoryTest.StackAlignment': [{
315      'elapsed_time_ms': 54000,
316      'losless_snippet': True,
317      'output_snippet': 'timed out',
318      'output_snippet_base64': '',
319      'status': 'FAILURE',
320    }],
321    'AlignedMemoryTest.StaticAlignment': [{
322      'elapsed_time_ms': 0,
323      'losless_snippet': True,
324      'output_snippet': '',
325      'output_snippet_base64': '',
326      'status': 'NOTRUN',
327    }],
328  }],
329  'swarming_summary': {
330    u'shards': [
331      {
332        u'state': u'COMPLETED',
333      },
334      {
335        u'state': u'TIMED_OUT',
336      },
337      ],
338  },
339  'test_locations': {
340    'AlignedMemoryTest.StackAlignment': {
341      'file': 'foo/bar/allocation_test.cc',
342      'line': 789,
343    },
344    'AlignedMemoryTest.StaticAlignment': {
345      'file': 'foo/bar/allocation_test.cc',
346      'line': 12,
347    },
348    'AlignedMemoryTest.DynamicAllocation': {
349      'file': 'foo/bar/allocation_test.cc',
350      'line': 123,
351    },
352    'AlignedMemoryTest.ScopedDynamicAllocation': {
353      'file': 'foo/bar/allocation_test.cc',
354      'line': 456,
355    },
356  },
357}
358
359
360class _StandardGtestMergeTest(unittest.TestCase):
361
362  def setUp(self):
363    self.temp_dir = tempfile.mkdtemp()
364
365  def tearDown(self):
366    shutil.rmtree(self.temp_dir)
367
368  def _write_temp_file(self, path, content):
369    abs_path = os.path.join(self.temp_dir, path.replace('/', os.sep))
370    if not os.path.exists(os.path.dirname(abs_path)):
371      os.makedirs(os.path.dirname(abs_path))
372    with open(abs_path, 'w') as f:
373      if isinstance(content, dict):
374        json.dump(content, f)
375      else:
376        assert isinstance(content, str)
377        f.write(content)
378    return abs_path
379
380
381class LoadShardJsonTest(_StandardGtestMergeTest):
382
383  def test_double_digit_jsons(self):
384    jsons_to_merge = []
385    for i in range(15):
386      json_dir = os.path.join(self.temp_dir, str(i))
387      json_path = os.path.join(json_dir, 'output.json')
388      if not os.path.exists(json_dir):
389        os.makedirs(json_dir)
390      with open(json_path, 'w') as f:
391        json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
392      jsons_to_merge.append(json_path)
393
394    content, err = standard_gtest_merge.load_shard_json(
395      0, None, jsons_to_merge)
396    self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content)
397    self.assertIsNone(err)
398
399    content, err = standard_gtest_merge.load_shard_json(
400      12, None, jsons_to_merge)
401    self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content)
402    self.assertIsNone(err)
403
404  def test_double_task_id_jsons(self):
405    jsons_to_merge = []
406    for i in range(15):
407      json_dir = os.path.join(self.temp_dir, 'deadbeef%d' % i)
408      json_path = os.path.join(json_dir, 'output.json')
409      if not os.path.exists(json_dir):
410        os.makedirs(json_dir)
411      with open(json_path, 'w') as f:
412        json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
413      jsons_to_merge.append(json_path)
414
415    content, err = standard_gtest_merge.load_shard_json(
416      0, 'deadbeef0', jsons_to_merge)
417    self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']},
418                     content)
419    self.assertIsNone(err)
420
421    content, err = standard_gtest_merge.load_shard_json(
422      12, 'deadbeef12', jsons_to_merge)
423    self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']},
424                     content)
425    self.assertIsNone(err)
426
427
428class MergeShardResultsTest(_StandardGtestMergeTest):
429  """Tests for merge_shard_results function."""
430
431  # pylint: disable=super-with-arguments
432  def setUp(self):
433    super(MergeShardResultsTest, self).setUp()
434    self.summary = None
435    self.test_files = []
436  # pylint: enable=super-with-arguments
437
438  def stage(self, summary, files):
439    self.summary = self._write_temp_file('summary.json', summary)
440    for path, content in files.items():
441      abs_path = self._write_temp_file(path, content)
442      self.test_files.append(abs_path)
443
444  def call(self):
445    stdout = six.StringIO()
446    with mock.patch('sys.stdout', stdout):
447      merged = standard_gtest_merge.merge_shard_results(
448          self.summary, self.test_files)
449      return merged, stdout.getvalue().strip()
450
451  def assertUnicodeEquals(self, expectation, result):
452    def convert_to_unicode(key_or_value):
453      if isinstance(key_or_value, str):
454        return six.text_type(key_or_value)
455      if isinstance(key_or_value, dict):
456        return {convert_to_unicode(k): convert_to_unicode(v)
457                for k, v in key_or_value.items()}
458      if isinstance(key_or_value, list):
459        return [convert_to_unicode(x) for x in key_or_value]
460      return key_or_value
461
462    unicode_expectations = convert_to_unicode(expectation)
463    unicode_result = convert_to_unicode(result)
464    self.assertEquals(unicode_expectations, unicode_result)
465
466  def test_ok(self):
467    # Two shards, both successfully finished.
468    self.stage({
469      u'shards': [
470        {
471          u'state': u'COMPLETED',
472        },
473        {
474          u'state': u'COMPLETED',
475        },
476      ],
477    },
478    {
479      '0/output.json': GOOD_GTEST_JSON_0,
480      '1/output.json': GOOD_GTEST_JSON_1,
481    })
482    merged, stdout = self.call()
483    merged['swarming_summary'] = {
484      'shards': [
485        {
486          u'state': u'COMPLETED',
487          u'outputs_ref': {
488            u'view_url': u'blah',
489          },
490        }
491      ],
492    }
493    self.assertUnicodeEquals(GOOD_GTEST_JSON_MERGED, merged)
494    self.assertEqual('', stdout)
495
496  def test_timed_out(self):
497    # Two shards, both successfully finished.
498    self.stage({
499      'shards': [
500        {
501          'state': 'COMPLETED',
502        },
503        {
504          'state': 'TIMED_OUT',
505        },
506      ],
507    },
508    {
509      '0/output.json': GOOD_GTEST_JSON_0,
510      '1/output.json': TIMED_OUT_GTEST_JSON_1,
511    })
512    merged, stdout = self.call()
513
514    self.assertUnicodeEquals(TIMED_OUT_GTEST_JSON_MERGED, merged)
515    self.assertIn(
516        'Test runtime exceeded allocated time\n', stdout)
517
518  def test_missing_summary_json(self):
519    # summary.json is missing, should return None and emit warning.
520    self.summary = os.path.join(self.temp_dir, 'summary.json')
521    merged, output = self.call()
522    self.assertEqual(None, merged)
523    self.assertIn('@@@STEP_WARNINGS@@@', output)
524    self.assertIn('summary.json is missing or can not be read', output)
525
526  def test_unfinished_shards(self):
527    # Only one shard (#1) finished. Shard #0 did not.
528    self.stage({
529      u'shards': [
530        None,
531        {
532          u'state': u'COMPLETED',
533        },
534      ],
535    },
536    {
537      u'1/output.json': GOOD_GTEST_JSON_1,
538    })
539    merged, stdout = self.call()
540    merged.pop('swarming_summary')
541    self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
542    self.assertIn(
543        '@@@STEP_WARNINGS@@@\nsome shards did not complete: 0\n', stdout)
544    self.assertIn(
545        '@@@STEP_LOG_LINE@some shards did not complete: 0@'
546        'Missing results from the following shard(s): 0@@@\n', stdout)
547
548  def test_missing_output_json(self):
549    # Shard #0 output json is missing.
550    self.stage({
551      u'shards': [
552        {
553          u'state': u'COMPLETED',
554        },
555        {
556          u'state': u'COMPLETED',
557        },
558      ],
559    },
560    {
561      u'1/output.json': GOOD_GTEST_JSON_1,
562    })
563    merged, stdout = self.call()
564    merged.pop('swarming_summary')
565    self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
566    self.assertIn(
567        'No result was found: '
568        'shard 0 test output was missing', stdout)
569
570  def test_large_output_json(self):
571    # a shard is too large.
572    self.stage({
573      u'shards': [
574        {
575          u'state': u'COMPLETED',
576        },
577        {
578          u'state': u'COMPLETED',
579        },
580      ],
581    },
582    {
583      '0/output.json': GOOD_GTEST_JSON_0,
584      '1/output.json': GOOD_GTEST_JSON_1,
585    })
586    old_json_limit = standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT
587    len0 = len(json.dumps(GOOD_GTEST_JSON_0))
588    len1 = len(json.dumps(GOOD_GTEST_JSON_1))
589    large_shard = "0" if len0 > len1 else "1"
590    try:
591      # Override max output.json size just for this test.
592      standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = min(len0,len1)
593      merged, stdout = self.call()
594      merged.pop('swarming_summary')
595      self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
596      self.assertIn(
597          'No result was found: '
598          'shard %s test output exceeded the size limit' % large_shard, stdout)
599    finally:
600      standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = old_json_limit
601
602
603class CommandLineTest(common_merge_script_tests.CommandLineTest):
604
605  # pylint: disable=super-with-arguments
606  def __init__(self, methodName='runTest'):
607    super(CommandLineTest, self).__init__(methodName, standard_gtest_merge)
608  # pylint: enable=super-with-arguments
609
610
611if __name__ == '__main__':
612  logging.basicConfig(
613      level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
614  if '-v' in sys.argv:
615    unittest.TestCase.maxDiff = None
616  unittest.main()
617