Codebase list emscripten / e74e0af
Avoid writing even sanity.txt if FROZEN_CACHE is enabled (#13316) These improvements to the `FROZE_CACHE` setting allow for a pre-populated, read-only cache. In this scenario the cache is cannot ever be locked, and that we don't even write the sanity file. Users can still force a sanity check with `emcc --check` but no sanity file will ever be written. Fixes: #13369 Sam Clegg authored 3 years ago GitHub committed 3 years ago
4 changed file(s) with 61 addition(s) and 47 deletion(s). Raw diff Collapse all Expand all
+16
-16
tests/test_sanity.py less more
389389
390390 self.assertContained('hello from emcc with no config file', self.run_js('a.out.js'))
391391
392 def erase_cache(self):
393 Cache.erase()
392 def clear_cache(self):
393 self.run_process([EMCC, '--clear-cache'])
394394 self.assertCacheEmpty()
395395
396396 def assertCacheEmpty(self):
405405 BUILDING_MESSAGE = 'generating system library: %s'
406406
407407 restore_and_set_up()
408 self.erase_cache()
408 self.clear_cache()
409409
410410 # Building a file that *does* need something *should* trigger cache
411411 # generation, but only the first time
424424 # Manual cache clearing
425425 restore_and_set_up()
426426 self.ensure_cache()
427 self.assertTrue(os.path.exists(Cache.dirname))
427 self.assertExists(Cache.dirname)
428428 output = self.do([EMCC, '--clear-cache'])
429429 self.assertIn('clearing cache', output)
430430 self.assertIn(SANITY_MESSAGE, output)
437437 make_fake_clang(self.in_dir('fake', 'bin', 'clang'), EXPECTED_LLVM_VERSION)
438438 make_fake_llc(self.in_dir('fake', 'bin', 'llc'), 'got wasm32 backend! WebAssembly 32-bit')
439439 with env_modify({'EM_LLVM_ROOT': self.in_dir('fake', 'bin')}):
440 self.assertTrue(os.path.exists(Cache.dirname))
440 self.assertExists(Cache.dirname)
441441 output = self.do([EMCC])
442442 self.assertIn('clearing cache', output)
443443 self.assertCacheEmpty()
445445 # FROZEN_CACHE prevents cache clears, and prevents building
446446 def test_FROZEN_CACHE(self):
447447 restore_and_set_up()
448 self.erase_cache()
448 self.clear_cache()
449449 self.ensure_cache()
450 self.assertTrue(os.path.exists(Cache.dirname))
450 self.assertExists(Cache.dirname)
451451 # changing config file should not clear cache
452452 add_to_config('FROZEN_CACHE = True')
453453 self.do([EMCC])
454 self.assertTrue(os.path.exists(Cache.dirname))
454 self.assertExists(Cache.dirname)
455455 # building libraries is disallowed
456456 output = self.do([EMBUILDER, 'build', 'libemmalloc'])
457 self.assertIn('FROZEN_CACHE disallows building system libs', output)
457 self.assertContained('FROZEN_CACHE is set, but cache file is missing', output)
458458
459459 # Test that if multiple processes attempt to access or build stuff to the
460460 # cache on demand, that exactly one of the processes will, and the other
483483 num_times_libc_was_built += 1
484484
485485 # The cache directory must exist after the build
486 self.assertTrue(os.path.exists(cache_dir_name))
486 self.assertExists(cache_dir_name)
487487 # The cache directory must contain a built libc
488 self.assertTrue(os.path.exists(os.path.join(cache_dir_name, libname)))
488 self.assertExists(os.path.join(cache_dir_name, libname))
489489 # Exactly one child process should have triggered libc build!
490490 self.assertEqual(num_times_libc_was_built, 1)
491491
624624
625625 print('normal build')
626626 with env_modify({'EMCC_FORCE_STDLIBS': None}):
627 Cache.erase()
627 self.clear_cache()
628628 build()
629629 test()
630630
631631 print('wacky env vars, these should not mess our bootstrapping')
632632 with env_modify({'EMCC_FORCE_STDLIBS': '1'}):
633 Cache.erase()
633 self.clear_cache()
634634 build()
635635 test()
636636
637637 def test_vanilla(self):
638638 restore_and_set_up()
639 Cache.erase()
639 self.clear_cache()
640640
641641 def make_fake(report):
642642 with open(config_file, 'a') as f:
688688 def test_embuilder_wasm_backend(self):
689689 restore_and_set_up()
690690 # the --lto flag makes us build wasm-bc
691 self.do([EMCC, '--clear-cache'])
691 self.clear_cache()
692692 self.run_process([EMBUILDER, 'build', 'libemmalloc'])
693693 self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten'))
694 self.do([EMCC, '--clear-cache'])
694 self.clear_cache()
695695 self.run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
696696 self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'lto'))
697697
+15
-9
tools/cache.py less more
3535 self.filelock = filelock.FileLock(self.filelock_name)
3636
3737 def acquire_cache_lock(self):
38 if config.FROZEN_CACHE:
39 # Raise an exception here rather than exit_with_error since in practice this
40 # should never happen
41 raise Exception('Attempt to lock the cache but FROZEN_CACHE is set')
42
3843 if not self.EM_EXCLUSIVE_CACHE_ACCESS and self.acquired_count == 0:
3944 logger.debug('PID %s acquiring multiprocess file lock to Emscripten cache at %s' % (str(os.getpid()), self.dirname))
4045 try:
113118 self.erase_file(self.get_lib_name(name))
114119
115120 def erase_file(self, shortname):
116 name = os.path.join(self.dirname, shortname)
117 if os.path.exists(name):
118 logger.info('deleting cached file: %s', name)
119 tempfiles.try_delete(name)
121 with self.lock():
122 name = os.path.join(self.dirname, shortname)
123 if os.path.exists(name):
124 logger.info('deleting cached file: %s', name)
125 tempfiles.try_delete(name)
120126
121127 def get_lib(self, libname, *args, **kwargs):
122128 name = self.get_lib_name(libname)
132138 if os.path.exists(cachename) and not force:
133139 return cachename
134140
141 if config.FROZEN_CACHE:
142 # Raise an exception here rather than exit_with_error since in practice this
143 # should never happen
144 raise Exception('FROZEN_CACHE is set, but cache file is missing: %s' % shortname)
145
135146 with self.lock():
136147 if os.path.exists(cachename) and not force:
137148 return cachename
138 # it doesn't exist yet, create it
139 if config.FROZEN_CACHE:
140 # it's ok to build small .txt marker files like "vanilla"
141 if not shortname.endswith('.txt'):
142 raise Exception('FROZEN_CACHE disallows building system libs: %s' % shortname)
143149 if what is None:
144150 if shortname.endswith(('.bc', '.so', '.a')):
145151 what = 'system library'
+1
-1
tools/config.py less more
7474 JS_ENGINES = [listify(engine) for engine in JS_ENGINES]
7575 WASM_ENGINES = [listify(engine) for engine in WASM_ENGINES]
7676 if not CACHE:
77 if root_is_writable():
77 if FROZEN_CACHE or root_is_writable():
7878 CACHE = path_from_root('cache')
7979 else:
8080 # Use the legacy method of putting the cache in the user's home directory
+29
-21
tools/shared.py less more
241241
242242
243243 def perform_sanity_checks():
244 # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
245 check_node_version()
246 check_llvm_version()
247
248 llvm_ok = check_llvm()
249
250 if os.environ.get('EM_IGNORE_SANITY'):
251 logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
252 return
253
244254 logger.info('(Emscripten: Running sanity checks)')
255
256 if not llvm_ok:
257 exit_with_error('failing sanity checks due to previous llvm failure')
245258
246259 with ToolchainProfiler.profile_block('sanity compiler_engine'):
247260 try:
266279 """
267280 if not force and os.environ.get('EMCC_SKIP_SANITY_CHECK') == '1':
268281 return
282
269283 # We set EMCC_SKIP_SANITY_CHECK so that any subprocesses that we launch will
270284 # not re-run the tests.
271285 os.environ['EMCC_SKIP_SANITY_CHECK'] = '1'
286
287 if config.FROZEN_CACHE:
288 if force:
289 perform_sanity_checks()
290 return
291
292 if os.environ.get('EM_IGNORE_SANITY'):
293 perform_sanity_checks()
294 return
295
272296 with ToolchainProfiler.profile_block('sanity'):
273 check_llvm_version()
274297 if not config.config_file:
275298 return # config stored directly in EM_CONFIG => skip sanity checks
276299 expected = generate_sanity()
282305 if sanity_data != expected:
283306 logger.debug('old sanity: %s' % sanity_data)
284307 logger.debug('new sanity: %s' % expected)
285 if config.FROZEN_CACHE:
286 logger.info('(Emscripten: config changed, cache may need to be cleared, but FROZEN_CACHE is set)')
287 else:
288 logger.info('(Emscripten: config changed, clearing cache)')
289 Cache.erase()
290 # the check actually failed, so definitely write out the sanity file, to
291 # avoid others later seeing failures too
292 force = False
308 logger.info('(Emscripten: config changed, clearing cache)')
309 Cache.erase()
310 # the check actually failed, so definitely write out the sanity file, to
311 # avoid others later seeing failures too
312 force = False
293313 else:
294314 if force:
295315 logger.debug(f'sanity file up-to-date but check forced: {sanity_file}')
298318 return # all is well
299319 else:
300320 logger.debug(f'sanity file not found: {sanity_file}')
301
302 # some warning, mostly not fatal checks - do them even if EM_IGNORE_SANITY is on
303 check_node_version()
304
305 llvm_ok = check_llvm()
306
307 if os.environ.get('EM_IGNORE_SANITY'):
308 logger.info('EM_IGNORE_SANITY set, ignoring sanity checks')
309 return
310
311 if not llvm_ok:
312 exit_with_error('failing sanity checks due to previous llvm failure')
313321
314322 perform_sanity_checks()
315323