1  /*
2   * Copyright (C) 2020 The Android Open Source Project
3   *
4   * Licensed under the Apache License, Version 2.0 (the "License");
5   * you may not use this file except in compliance with the License.
6   * You may obtain a copy of the License at
7   *
8   *      http://www.apache.org/licenses/LICENSE-2.0
9   *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   */
16  
17  #include <android-base/logging.h>
18  #include <android-base/properties.h>
19  #include <android-base/unique_fd.h>
20  #include <ftw.h>
21  #include <gtest/gtest.h>
22  #include <libgen.h>
23  #include <stdlib.h>
24  #include <unistd.h>
25  
26  #include <algorithm>
27  #include <cassert>
28  #include <cmath>
29  #include <fstream>
30  #include <iostream>
31  #include <map>
32  #include <memory>
33  #include <random>
34  #include <set>
35  #include <string>
36  #include <thread>
37  #include <utility>
38  #include <vector>
39  
40  #include "GeneratedTestUtils.h"
41  #include "SupportLibraryTestUtils.h"
42  #include "SupportLibraryWrapper.h"
43  #include "TmpDirectoryUtils.h"
44  
45  // Systrace is not available from CTS tests due to platform layering
46  // constraints. We reuse the NNTEST_ONLY_PUBLIC_API flag, as that should also be
47  // the case for CTS (public APIs only).
48  #ifndef NNTEST_ONLY_PUBLIC_API
49  #include <Tracing.h>
50  #else
51  #define NNTRACE_FULL_RAW(...)
52  #define NNTRACE_APP(...)
53  #define NNTRACE_APP_SWITCH(...)
54  #endif
55  
56  const char* kQCDspLoadPathEnv = "ADSP_LIBRARY_PATH";
57  
58  extern std::string SUPPORT_LIBRARY_NAME;
59  
60  namespace android::nn::generated_tests {
61  using namespace sl_wrapper;
62  using namespace test_helper;
63  
64  enum ComputeWithDeviceMemoriesResult {
65      SKIP,
66      OK,
67  };
68  
69  class GeneratedTests : public GeneratedTestBase {
70     protected:
71      void SetUp() override;
72      void TearDown() override;
73  
74      bool shouldSkipTest();
75  
76      ANeuralNetworksMemory* createDeviceMemoryForInput(const Compilation& compilation,
77                                                        uint32_t index);
78      ANeuralNetworksMemory* createDeviceMemoryForOutput(const Compilation& compilation,
79                                                         uint32_t index);
80      ComputeWithDeviceMemoriesResult computeWithDeviceMemories(
81              const Compilation& compilation, const TestModel& testModel, Execution* execution,
82              Execution::ComputeMode computeMode, Result* result, std::vector<TestBuffer>* outputs);
83      bool checkSupported(const Model& model, ANeuralNetworksDevice* device);
84      std::optional<Compilation> compileModel(const Model& model, ANeuralNetworksDevice* device);
85      void executeWithCompilation(const Compilation& compilation, const TestModel& testModel);
86      void executeOnce(const Model& model, const TestModel& testModel);
87      void executeMultithreadedOwnCompilation(const Model& model, const TestModel& testModel);
88      void executeMultithreadedSharedCompilation(const Model& model, const TestModel& testModel);
89      // Test driver for those generated from ml/nn/runtime/test/spec
90      void execute(const TestModel& testModel);
91  
92      // VNDK version of the device under test.
93      static int mVndkVersion;
94  
95      std::string mCacheDir;
96      std::vector<uint8_t> mToken;
97      bool mTestCompilationCaching = false;
98      bool mTestDynamicOutputShape = false;
99      bool mExpectFailure = false;
100      bool mTestQuantizationCoupling = false;
101      bool mTestDeviceMemory = false;
102      Execution::ComputeMode mComputeMode = Execution::getComputeMode();
103  
104      std::unique_ptr<const NnApiSupportLibrary> mNnApi;
105  };
106  
107  int GeneratedTests::mVndkVersion = __ANDROID_API_FUTURE__;
108  
109  // Tag for the dynamic output shape tests
110  class DynamicOutputShapeTest : public GeneratedTests {
111     protected:
DynamicOutputShapeTest()112      DynamicOutputShapeTest() { mTestDynamicOutputShape = true; }
113  };
114  
115  // Tag for the fenced execute tests
116  class FencedComputeTest : public GeneratedTests {};
117  
118  // Tag for the generated validation tests
119  class GeneratedValidationTests : public GeneratedTests {
120     protected:
GeneratedValidationTests()121      GeneratedValidationTests() { mExpectFailure = true; }
122  };
123  
124  class QuantizationCouplingTest : public GeneratedTests {
125     protected:
QuantizationCouplingTest()126      QuantizationCouplingTest() { mTestQuantizationCoupling = true; }
127  };
128  
129  class DeviceMemoryTest : public GeneratedTests {
130     protected:
DeviceMemoryTest()131      DeviceMemoryTest() { mTestDeviceMemory = true; }
132  };
133  
checkSupported(const Model & model,ANeuralNetworksDevice * device)134  bool GeneratedTests::checkSupported(const Model& model, ANeuralNetworksDevice* device) {
135      constexpr static int MAX_NUM_OPS = 256;
136      std::array<bool, MAX_NUM_OPS> supportedOps;
137      for (int i = 0; i < MAX_NUM_OPS; ++i) {
138          supportedOps[i] = true;
139      }
140      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksModel_getSupportedOperationsForDevices(
141                        model.getHandle(), &device, /*numDevices=*/1, supportedOps.data()),
142                ANEURALNETWORKS_NO_ERROR);
143      const bool fullySupportedModel =
144              std::all_of(supportedOps.begin(), supportedOps.end(), [](bool v) { return v; });
145      return fullySupportedModel;
146  }
147  
createCacheFds(const std::vector<std::string> & files)148  static std::vector<base::unique_fd> createCacheFds(const std::vector<std::string>& files) {
149      std::vector<base::unique_fd> fds;
150      fds.reserve(files.size());
151      for (const auto& file : files) {
152          auto fd = base::unique_fd(open(file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR));
153          if (fd.get() == -1) {
154              [] { FAIL(); }();
155              return {};
156          }
157          fds.push_back(std::move(fd));
158      }
159      return fds;
160  }
161  
compileModel(const Model & model,ANeuralNetworksDevice * device)162  std::optional<Compilation> GeneratedTests::compileModel(const Model& model,
163                                                          ANeuralNetworksDevice* device) {
164      NNTRACE_APP(NNTRACE_PHASE_COMPILATION, "compileModel");
165  
166      if (mTestCompilationCaching) {
167          // Compile the model twice with the same token, so that compilation caching will be
168          // exercised if supported by the driver.
169          // No invalid model will be passed to this branch.
170          EXPECT_FALSE(mExpectFailure);
171  
172          std::string mode = ::android::base::GetProperty("debug.nn.slts.caching", "random");
173          bool useSetCachingFromFds;
174          if (mode == "path") {
175              useSetCachingFromFds = false;
176          } else if (mode == "fds") {
177              useSetCachingFromFds = true;
178          } else if (mode == "random") {
179              std::string testName = ::testing::UnitTest::GetInstance()->current_test_info()->name();
180              std::seed_seq seq(testName.begin(), testName.end());
181              std::mt19937 gen(seq);
182              std::bernoulli_distribution d(0.5);
183              useSetCachingFromFds = d(gen);
184          } else {
185              [&mode] {
186                  FAIL() << "System property debug.nn.slts.caching should be one of \"path\", "
187                            "\"fds\", or \"random\"; got \""
188                         << mode << "\"";
189              }();
190              return {};
191          }
192          SCOPED_TRACE("Use setCachingFromFds = " + std::to_string(useSetCachingFromFds) + " (" +
193                       mode + ")");
194          std::cout << "\nUse setCachingFromFds = " << std::boolalpha << useSetCachingFromFds << " ("
195                    << mode << ")" << std::endl;
196  
197          std::vector<std::string> modelCacheFilenames, dataCacheFilenames;
198          if (useSetCachingFromFds) {
199              uint32_t numModelCacheFiles, numDataCacheFiles;
200              EXPECT_EQ(mNnApi->getFL5()->SL_ANeuralNetworksDevice_getNumberOfCacheFilesNeeded(
201                                device, &numModelCacheFiles, &numDataCacheFiles),
202                        ANEURALNETWORKS_NO_ERROR);
203              for (uint32_t i = 0; i < numModelCacheFiles; i++) {
204                  modelCacheFilenames.push_back({mCacheDir + "/model" + std::to_string(i)});
205              }
206              for (uint32_t i = 0; i < numDataCacheFiles; i++) {
207                  dataCacheFilenames.push_back({mCacheDir + "/data" + std::to_string(i)});
208              }
209          }
210  
211          auto resultCompilation1 = Compilation::createForDevice(mNnApi.get(), &model, device);
212          EXPECT_EQ(resultCompilation1.first, Result::NO_ERROR);
213          auto compilation1 = std::move(resultCompilation1.second);
214          if (useSetCachingFromFds) {
215              auto modelCacheFds = createCacheFds(modelCacheFilenames);
216              auto dataCacheFds = createCacheFds(dataCacheFilenames);
217              EXPECT_EQ(compilation1.setCachingFromFds(modelCacheFds, dataCacheFds, mToken),
218                        Result::NO_ERROR);
219          } else {
220              EXPECT_EQ(compilation1.setCaching(mCacheDir, mToken), Result::NO_ERROR);
221          }
222          EXPECT_EQ(compilation1.finish(), Result::NO_ERROR);
223  
224          auto resultCompilation2 = Compilation::createForDevice(mNnApi.get(), &model, device);
225          EXPECT_EQ(resultCompilation2.first, Result::NO_ERROR);
226          auto compilation2 = std::move(resultCompilation2.second);
227          if (useSetCachingFromFds) {
228              auto modelCacheFds = createCacheFds(modelCacheFilenames);
229              auto dataCacheFds = createCacheFds(dataCacheFilenames);
230              EXPECT_EQ(compilation2.setCachingFromFds(modelCacheFds, dataCacheFds, mToken),
231                        Result::NO_ERROR);
232          } else {
233              EXPECT_EQ(compilation2.setCaching(mCacheDir, mToken), Result::NO_ERROR);
234          }
235          EXPECT_EQ(compilation2.finish(), Result::NO_ERROR);
236  
237          return compilation2;
238      } else {
239          auto resultCompilation = Compilation::createForDevice(mNnApi.get(), &model, device);
240          EXPECT_EQ(resultCompilation.first, Result::NO_ERROR);
241          auto compilation = std::move(resultCompilation.second);
242          Result result = compilation.finish();
243  
244          // For valid model, we check the compilation result == NO_ERROR.
245          // For invalid model, the driver may fail at compilation or execution, so any result code is
246          // permitted at this point.
247          if (mExpectFailure && result != Result::NO_ERROR) return std::nullopt;
248          EXPECT_EQ(result, Result::NO_ERROR);
249          return compilation;
250      }
251  }
252  
computeWithPtrs(const TestModel & testModel,Execution * execution,Execution::ComputeMode computeMode,Result * result,std::vector<TestBuffer> * outputs)253  void computeWithPtrs(const TestModel& testModel, Execution* execution,
254                       Execution::ComputeMode computeMode, Result* result,
255                       std::vector<TestBuffer>* outputs) {
256      {
257          NNTRACE_APP(NNTRACE_PHASE_INPUTS_AND_OUTPUTS, "computeWithPtrs example");
258          createRequest(testModel, execution, outputs);
259      }
260      *result = execution->compute(computeMode);
261  }
262  
createDeviceMemoryForInput(const Compilation & compilation,uint32_t index)263  ANeuralNetworksMemory* GeneratedTests::createDeviceMemoryForInput(const Compilation& compilation,
264                                                                    uint32_t index) {
265      ANeuralNetworksMemoryDesc* desc = nullptr;
266      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemoryDesc_create(&desc), ANEURALNETWORKS_NO_ERROR);
267      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemoryDesc_addInputRole(
268                        desc, compilation.getHandle(), index, 1.0f),
269                ANEURALNETWORKS_NO_ERROR);
270      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemoryDesc_finish(desc), ANEURALNETWORKS_NO_ERROR);
271      ANeuralNetworksMemory* memory = nullptr;
272      mNnApi->getFL5()->ANeuralNetworksMemory_createFromDesc(desc, &memory);
273      mNnApi->getFL5()->ANeuralNetworksMemoryDesc_free(desc);
274      return memory;
275  }
276  
createDeviceMemoryForOutput(const Compilation & compilation,uint32_t index)277  ANeuralNetworksMemory* GeneratedTests::createDeviceMemoryForOutput(const Compilation& compilation,
278                                                                     uint32_t index) {
279      ANeuralNetworksMemoryDesc* desc = nullptr;
280      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemoryDesc_create(&desc), ANEURALNETWORKS_NO_ERROR);
281      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemoryDesc_addOutputRole(
282                        desc, compilation.getHandle(), index, 1.0f),
283                ANEURALNETWORKS_NO_ERROR);
284      EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemoryDesc_finish(desc), ANEURALNETWORKS_NO_ERROR);
285      ANeuralNetworksMemory* memory = nullptr;
286      mNnApi->getFL5()->ANeuralNetworksMemory_createFromDesc(desc, &memory);
287      mNnApi->getFL5()->ANeuralNetworksMemoryDesc_free(desc);
288      return memory;
289  }
290  
291  // Set result = Result::NO_ERROR and outputs = {} if the test should be skipped.
computeWithDeviceMemories(const Compilation & compilation,const TestModel & testModel,Execution * execution,Execution::ComputeMode computeMode,Result * result,std::vector<TestBuffer> * outputs)292  ComputeWithDeviceMemoriesResult GeneratedTests::computeWithDeviceMemories(
293          const Compilation& compilation, const TestModel& testModel, Execution* execution,
294          Execution::ComputeMode computeMode, Result* result, std::vector<TestBuffer>* outputs) {
295      EXPECT_NE(execution, nullptr);
296      EXPECT_NE(result, nullptr);
297      EXPECT_NE(outputs, nullptr);
298      outputs->clear();
299      std::vector<Memory> inputMemories, outputMemories;
300  
301      {
302          NNTRACE_APP(NNTRACE_PHASE_INPUTS_AND_OUTPUTS, "computeWithDeviceMemories example");
303          // Model inputs.
304          for (uint32_t i = 0; i < testModel.main.inputIndexes.size(); i++) {
305              SCOPED_TRACE("Input index: " + std::to_string(i));
306              const auto& operand = testModel.main.operands[testModel.main.inputIndexes[i]];
307              // Omitted input.
308              if (operand.data.size() == 0) {
309                  EXPECT_EQ(Result::NO_ERROR, execution->setInput(i, nullptr, 0));
310                  continue;
311              }
312  
313              // Create device memory.
314              ANeuralNetworksMemory* memory = createDeviceMemoryForInput(compilation, i);
315              if (memory == nullptr) {
316                  return ComputeWithDeviceMemoriesResult::SKIP;
317              }
318              auto& wrapperMemory = inputMemories.emplace_back(Memory(mNnApi.get(), memory));
319  
320              // Copy data from TestBuffer to device memory.
321              auto ashmem = TestAshmem::createFrom(mNnApi.get(), operand.data);
322              EXPECT_NE(ashmem, nullptr);
323              EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemory_copy(ashmem->get()->get(), memory),
324                        ANEURALNETWORKS_NO_ERROR);
325              EXPECT_EQ(Result::NO_ERROR, execution->setInputFromMemory(i, &wrapperMemory, 0, 0));
326          }
327  
328          // Model outputs.
329          for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) {
330              SCOPED_TRACE("Output index: " + std::to_string(i));
331              ANeuralNetworksMemory* memory = createDeviceMemoryForOutput(compilation, i);
332              if (memory == nullptr) {
333                  return ComputeWithDeviceMemoriesResult::SKIP;
334              }
335              auto& wrapperMemory = outputMemories.emplace_back(Memory(mNnApi.get(), memory));
336              EXPECT_EQ(Result::NO_ERROR, execution->setOutputFromMemory(i, &wrapperMemory, 0, 0));
337          }
338      }
339  
340      *result = execution->compute(computeMode);
341  
342      // Copy out output results.
343      for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) {
344          SCOPED_TRACE("Output index: " + std::to_string(i));
345          const auto& operand = testModel.main.operands[testModel.main.outputIndexes[i]];
346          const size_t bufferSize = operand.data.size();
347          auto& output = outputs->emplace_back(bufferSize);
348  
349          auto ashmem = TestAshmem::createFrom(mNnApi.get(), output);
350          EXPECT_NE(ashmem, nullptr);
351          EXPECT_EQ(mNnApi->getFL5()->ANeuralNetworksMemory_copy(outputMemories[i].get(),
352                                                                 ashmem->get()->get()),
353                    ANEURALNETWORKS_NO_ERROR);
354          std::copy(ashmem->dataAs<uint8_t>(), ashmem->dataAs<uint8_t>() + bufferSize,
355                    output.getMutable<uint8_t>());
356      }
357      return ComputeWithDeviceMemoriesResult::OK;
358  }
359  
executeWithCompilation(const Compilation & compilation,const TestModel & testModel)360  void GeneratedTests::executeWithCompilation(const Compilation& compilation,
361                                              const TestModel& testModel) {
362      NNTRACE_APP(NNTRACE_PHASE_EXECUTION, "executeWithCompilation example");
363  
364      Execution execution(mNnApi.get(), &compilation);
365      Result result;
366      std::vector<TestBuffer> outputs;
367  
368      if (mTestDeviceMemory) {
369          if (computeWithDeviceMemories(compilation, testModel, &execution, mComputeMode, &result,
370                                        &outputs) == ComputeWithDeviceMemoriesResult::SKIP) {
371              std::cout << "\nModel not supported by device memories. Skipping" << std::endl;
372              return;
373          }
374      } else {
375          computeWithPtrs(testModel, &execution, mComputeMode, &result, &outputs);
376      }
377  
378      if (result == Result::NO_ERROR && outputs.empty()) {
379          return;
380      }
381  
382      {
383          NNTRACE_APP(NNTRACE_PHASE_RESULTS, "executeWithCompilation example");
384          if (mExpectFailure) {
385              ASSERT_NE(result, Result::NO_ERROR);
386              return;
387          } else {
388              ASSERT_EQ(result, Result::NO_ERROR);
389          }
390  
391          // Check output dimensions.
392          for (uint32_t i = 0; i < testModel.main.outputIndexes.size(); i++) {
393              SCOPED_TRACE("Output index: " + std::to_string(i));
394              const auto& output = testModel.main.operands[testModel.main.outputIndexes[i]];
395              if (output.isIgnored) continue;
396              std::vector<uint32_t> actualDimensions;
397              ASSERT_EQ(Result::NO_ERROR, execution.getOutputOperandDimensions(i, &actualDimensions));
398              ASSERT_EQ(output.dimensions, actualDimensions);
399          }
400  
401          checkResults(testModel, outputs);
402      }
403  }
404  
executeOnce(const Model & model,const TestModel & testModel)405  void GeneratedTests::executeOnce(const Model& model, const TestModel& testModel) {
406      NNTRACE_APP(NNTRACE_PHASE_OVERALL, "executeOnce");
407      uint32_t numDevices = 0;
408      mNnApi->getFL5()->ANeuralNetworks_getDeviceCount(&numDevices);
409      bool modelSupported = false;
410      for (uint32_t i = 0; i < numDevices; ++i) {
411          ANeuralNetworksDevice* device = nullptr;
412          mNnApi->getFL5()->ANeuralNetworks_getDevice(i, &device);
413          const char* deviceName = nullptr;
414          mNnApi->getFL5()->ANeuralNetworksDevice_getName(device, &deviceName);
415          SCOPED_TRACE("Device = " + std::string(deviceName));
416          std::cout << "\nDevice = " << deviceName << std::endl;
417          if (!checkSupported(model, device)) {
418              std::cout << "\nModel not supported by device " << deviceName << ". Skipping"
419                        << std::endl;
420              continue;
421          }
422          modelSupported = true;
423          std::cout << "\nModel supported" << std::endl;
424          std::optional<Compilation> compilation = compileModel(model, device);
425          // Early return if compilation fails. The compilation result code is
426          // checked in compileModel.
427          if (!compilation) return;
428          executeWithCompilation(compilation.value(), testModel);
429          std::cout << "\nExecution completed" << std::endl;
430      }
431      if (!modelSupported) {
432          std::cout << "\nModel not supported by any device\n"
433                    << "SKIPPED" << std::endl;
434      }
435  }
436  
executeMultithreadedOwnCompilation(const Model & model,const TestModel & testModel)437  void GeneratedTests::executeMultithreadedOwnCompilation(const Model& model,
438                                                          const TestModel& testModel) {
439      NNTRACE_APP(NNTRACE_PHASE_OVERALL, "executeMultithreadedOwnCompilation");
440      SCOPED_TRACE("MultithreadedOwnCompilation");
441      std::cout << "\nMultithreadedOwnCompilation" << std::endl;
442      std::vector<std::thread> threads;
443      for (int i = 0; i < 10; i++) {
444          threads.push_back(std::thread([&]() { executeOnce(model, testModel); }));
445      }
446      std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); });
447  }
448  
executeMultithreadedSharedCompilation(const Model & model,const TestModel & testModel)449  void GeneratedTests::executeMultithreadedSharedCompilation(const Model& model,
450                                                             const TestModel& testModel) {
451      NNTRACE_APP(NNTRACE_PHASE_OVERALL, "executeMultithreadedSharedCompilation");
452      SCOPED_TRACE("MultithreadedSharedCompilation");
453      std::cout << "\nMultithreadedSharedCompilation" << std::endl;
454      uint32_t numDevices = 0;
455      mNnApi->getFL5()->ANeuralNetworks_getDeviceCount(&numDevices);
456      bool modelSupported = false;
457      for (uint32_t i = 0; i < numDevices; ++i) {
458          ANeuralNetworksDevice* device = nullptr;
459          mNnApi->getFL5()->ANeuralNetworks_getDevice(i, &device);
460          const char* deviceName = nullptr;
461          mNnApi->getFL5()->ANeuralNetworksDevice_getName(device, &deviceName);
462          SCOPED_TRACE("Device = " + std::string(deviceName));
463          std::cout << "\nDevice = " << deviceName << std::endl;
464          if (!checkSupported(model, device)) {
465              std::cout << "\nModel not supported by device " << deviceName << ". Skipping"
466                        << std::endl;
467              continue;
468          }
469          modelSupported = true;
470          std::cout << "\nModel supported" << std::endl;
471          std::optional<Compilation> compilation = compileModel(model, device);
472          // Early return if compilation fails. The ompilation result code is
473          // checked in compileModel.
474          if (!compilation) return;
475          std::vector<std::thread> threads;
476          for (int i = 0; i < 10; i++) {
477              threads.push_back(
478                      std::thread([&]() { executeWithCompilation(compilation.value(), testModel); }));
479          }
480          std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); });
481          std::cout << "\nExecution completed" << std::endl;
482      }
483      if (!modelSupported) {
484          std::cout << "\nModel not supported by any device\n"
485                    << "SKIPPED" << std::endl;
486      }
487  }
488  
489  // Test driver for those generated from ml/nn/runtime/test/spec
execute(const TestModel & testModel)490  void GeneratedTests::execute(const TestModel& testModel) {
491      NNTRACE_APP(NNTRACE_PHASE_OVERALL, "execute");
492      GeneratedModel model(mNnApi.get());
493      createModel(mNnApi.get(), testModel, mTestDynamicOutputShape, &model);
494      if (testModel.expectFailure && !model.isValid()) {
495          return;
496      }
497      ASSERT_EQ(model.finish(), Result::NO_ERROR);
498      ASSERT_TRUE(model.isValid());
499      auto executeInternal = [&testModel, &model, this]() {
500          SCOPED_TRACE("TestCompilationCaching = " + std::to_string(mTestCompilationCaching));
501          std::cout << "\nCompilationCaching = " << std::boolalpha << mTestCompilationCaching
502                    << std::endl;
503  #ifndef NNTEST_MULTITHREADED
504          executeOnce(model, testModel);
505  #else   // defined(NNTEST_MULTITHREADED)
506          executeMultithreadedOwnCompilation(model, testModel);
507          executeMultithreadedSharedCompilation(model, testModel);
508  #endif  // !defined(NNTEST_MULTITHREADED)
509      };
510      mTestCompilationCaching = false;
511      executeInternal();
512      if (!mExpectFailure) {
513          mTestCompilationCaching = true;
514          executeInternal();
515      }
516  }
517  
shouldSkipTest()518  bool GeneratedTests::shouldSkipTest() {
519      // A map of {min VNDK version -> tests that should be skipped with earlier VNDK versions}.
520      // The listed tests are added in a later release, but exercising old APIs. They should be
521      // skipped if the device has a mixed build of system and vendor partitions.
522      static const std::map<int, std::set<std::string>> kMapOfMinVndkVersionToTests = {
523              {
524                      __ANDROID_API_R__,
525                      {
526                              "add_broadcast_quant8_all_inputs_as_internal",
527                      },
528              },
529      };
530      for (const auto& [minVersion, names] : kMapOfMinVndkVersionToTests) {
531          if (mVndkVersion < minVersion && names.count(kTestName) > 0) {
532              return true;
533          }
534      }
535      return false;
536  }
537  
SetUp()538  void GeneratedTests::SetUp() {
539      const char* libdir = dirname(SUPPORT_LIBRARY_NAME.c_str());
540      setenv(kQCDspLoadPathEnv, libdir, 1);
541      LOG(INFO) << "Overwritten system env variable " << kQCDspLoadPathEnv << " with " << libdir;
542      mNnApi = loadNnApiSupportLibrary(SUPPORT_LIBRARY_NAME);
543  
544      GeneratedTestBase::SetUp();
545  
546      mVndkVersion = ::android::base::GetIntProperty("ro.vndk.version", __ANDROID_API_FUTURE__);
547      if (shouldSkipTest()) {
548          GTEST_SKIP();
549          return;
550      }
551  
552      char cacheDirTemp[] = NN_TMP_DIR "/TestCompilationCachingXXXXXX";
553      char* cacheDir = mkdtemp(cacheDirTemp);
554      ASSERT_NE(cacheDir, nullptr);
555      mCacheDir = cacheDir;
556      mToken = std::vector<uint8_t>(ANEURALNETWORKS_BYTE_SIZE_OF_CACHE_TOKEN, 0);
557  }
558  
TearDown()559  void GeneratedTests::TearDown() {
560      mNnApi.reset(nullptr);
561  
562      if (!::testing::Test::HasFailure()) {
563          // TODO: Switch to std::filesystem::remove_all once libc++fs is made available in CTS.
564          // Remove the cache directory specified by path recursively.
565          auto callback = [](const char* child, const struct stat*, int, struct FTW*) {
566              return remove(child);
567          };
568          nftw(mCacheDir.c_str(), callback, 128, FTW_DEPTH | FTW_MOUNT | FTW_PHYS);
569      }
570      GeneratedTestBase::TearDown();
571  }
572  
573  #ifdef NNTEST_COMPUTE_MODE
TEST_P(GeneratedTests,Sync)574  TEST_P(GeneratedTests, Sync) {
575      std::cout << "\nComputeMode = SYNC" << std::endl;
576      mComputeMode = Execution::ComputeMode::SYNC;
577      execute(testModel);
578  }
579  
TEST_P(GeneratedTests,Burst)580  TEST_P(GeneratedTests, Burst) {
581      std::cout << "\nComputeMode = BURST" << std::endl;
582      mComputeMode = Execution::ComputeMode::BURST;
583      execute(testModel);
584  }
585  #else
TEST_P(GeneratedTests,Test)586  TEST_P(GeneratedTests, Test) {
587      execute(testModel);
588  }
589  #endif
590  
TEST_P(DynamicOutputShapeTest,Test)591  TEST_P(DynamicOutputShapeTest, Test) {
592      execute(testModel);
593  }
594  
TEST_P(GeneratedValidationTests,Test)595  TEST_P(GeneratedValidationTests, Test) {
596      execute(testModel);
597  }
598  
TEST_P(QuantizationCouplingTest,Test)599  TEST_P(QuantizationCouplingTest, Test) {
600      execute(convertQuant8AsymmOperandsToSigned(testModel));
601  }
602  
TEST_P(DeviceMemoryTest,Test)603  TEST_P(DeviceMemoryTest, Test) {
604      execute(testModel);
605  }
606  
TEST_P(FencedComputeTest,Test)607  TEST_P(FencedComputeTest, Test) {
608      mComputeMode = Execution::ComputeMode::FENCED;
609      execute(testModel);
610  }
611  
612  INSTANTIATE_GENERATED_TEST(GeneratedTests,
__anon5ae799d20a02(const TestModel& testModel) 613                             [](const TestModel& testModel) { return !testModel.expectFailure; });
614  
__anon5ae799d20b02(const TestModel& testModel) 615  INSTANTIATE_GENERATED_TEST(DynamicOutputShapeTest, [](const TestModel& testModel) {
616      return !testModel.expectFailure && !testModel.hasScalarOutputs();
617  });
618  
__anon5ae799d20c02(const TestModel& testModel) 619  INSTANTIATE_GENERATED_TEST(GeneratedValidationTests, [](const TestModel& testModel) {
620      return testModel.expectFailure && !testModel.isInfiniteLoopTimeoutTest();
621  });
622  
__anon5ae799d20d02(const TestModel& testModel) 623  INSTANTIATE_GENERATED_TEST(QuantizationCouplingTest, [](const TestModel& testModel) {
624      return !testModel.expectFailure && testModel.main.operations.size() == 1 &&
625             testModel.referenced.size() == 0 && testModel.hasQuant8CoupledOperands();
626  });
627  
__anon5ae799d20e02(const TestModel& testModel) 628  INSTANTIATE_GENERATED_TEST(DeviceMemoryTest, [](const TestModel& testModel) {
629      return !testModel.expectFailure &&
630             std::all_of(testModel.main.outputIndexes.begin(), testModel.main.outputIndexes.end(),
631                         [&testModel](uint32_t index) {
632                             return testModel.main.operands[index].data.size() > 0;
633                         });
634  });
635  
__anon5ae799d21002(const TestModel& testModel) 636  INSTANTIATE_GENERATED_TEST(FencedComputeTest, [](const TestModel& testModel) {
637      return !testModel.expectFailure &&
638             std::all_of(testModel.main.outputIndexes.begin(), testModel.main.outputIndexes.end(),
639                         [&testModel](uint32_t index) {
640                             return testModel.main.operands[index].data.size() > 0;
641                         });
642  });
643  
644  }  // namespace android::nn::generated_tests
645