James Henderson via llvm-dev
2018-Jul-25 09:11 UTC
[llvm-dev] are the LLD libraries thread safe?
Hi Andrew, LLD relies on various bits of global state which are manipulated during the link, so I wouldn't expect it to be thread safe at that level, although it does attempt to reset that global state at the start of each call to link(), so it should be callable sequentially. Regards, James On 25 July 2018 at 02:37, Andrew Kelley via llvm-dev < llvm-dev at lists.llvm.org> wrote:> Follow-up - > > Putting a global lock on running LLD indeed solved the issue, so I'm > guessing that the LLD libraries do not in fact support being called from > multiple threads simultaneously. > > > On Tue, Jul 24, 2018 at 9:10 PM, Andrew Kelley <superjoe30 at gmail.com> > wrote: > >> E.g. Is it intended to be allowed to call lld::elf::link in 2 different >> threads at the same time? >> >> Follows is an example Valgrind error I ran into when doing the above. >> >> I'll try putting a global resource lock on invoking LLD and see if it >> solves the problem. >> >> ==5467== Invalid write of size 8 >> ==5467== at 0x525509: llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::moveFromOldBuckets(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>*, llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >> (DenseMap.h:375) >> ==5467== by 0x524C7F: llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::grow(unsigned int) (DenseMap.h:739) >> ==5467== by 0x524077: llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::grow(unsigned int) (DenseMap.h:502) >> ==5467== by 0x522E54: llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>* llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::InsertIntoBucketImpl<llvm::CachedHashStringRef>(llvm::CachedHashStringRef >> const&, llvm::CachedHashStringRef const&, llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>*) (DenseMap.h:546) >> ==5467== by 0x521258: llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>* llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::InsertIntoBucket<llvm::CachedHashStringRef, >> int>(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*, >> llvm::CachedHashStringRef&&, int&&) (DenseMap.h:512) >> ==5467== by 0x51DEC9: std::pair<llvm::DenseMapIterator<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>, false>, >> bool> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::try_emplace<int>(llvm::CachedHashStringRef&&, int&&) (DenseMap.h:215) >> ==5467== by 0x51C52F: llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::insert(std::pair<llvm::CachedHashStringRef, int>&&) (DenseMap.h:199) >> ==5467== by 0x516836: lld::elf::SymbolTable::insert(llvm::StringRef) >> (SymbolTable.cpp:225) >> ==5467== by 0x516A21: lld::elf::SymbolTable::insert(llvm::StringRef, >> unsigned char, unsigned char, bool, lld::elf::InputFile*) >> (SymbolTable.cpp:260) >> ==5467== by 0x51B88C: void lld::elf::SymbolTable::addShar >> ed<llvm::object::ELFType<(llvm::support::endianness)1, true> >> >(llvm::StringRef, lld::elf::SharedFile<llvm::obj >> ect::ELFType<(llvm::support::endianness)1, true> >&, >> llvm::object::ELFType<(llvm::support::endianness)1, true>::Sym const&, >> unsigned int, unsigned int) (SymbolTable.cpp:483) >> ==5467== by 0x4E762A: lld::elf::SharedFile<llvm::obj >> ect::ELFType<(llvm::support::endianness)1, true> >::parseRest() >> (InputFiles.cpp:895) >> ==5467== by 0x51F69F: void lld::elf::SymbolTable::addFile >> <llvm::object::ELFType<(llvm::support::endianness)1, true> >> >(lld::elf::InputFile*) (SymbolTable.cpp:99) >> ==5467== Address 0xc6c7e80 is 53,568 bytes inside a block of size 98,304 >> free'd >> ==5467== at 0x4C2E26B: operator delete(void*) (in >> /nix/store/j7sf6f2i9gws7c5mn8jyi8wc08ab1zbp-valgrind-3.13.0/ >> lib/valgrind/vgpreload_memcheck-amd64-linux.so) >> ==5467== by 0x4C714C: llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::~DenseMap() (DenseMap.h:681) >> ==5467== by 0x4C9C7B: lld::elf::SymbolTable::~SymbolTable() >> (SymbolTable.h:36) >> ==5467== by 0x4C9BF1: llvm::SpecificBumpPtrAllocator >> <lld::elf::SymbolTable>::DestroyAll()::{lambda(char*, >> char*)#1}::operator()(char*, char*) const (Allocator.h:410) >> ==5467== by 0x4C9D77: llvm::SpecificBumpPtrAllocator >> <lld::elf::SymbolTable>::DestroyAll() (Allocator.h:421) >> ==5467== by 0x4D218F: lld::SpecificAlloc<lld::elf::SymbolTable>::reset() >> (Memory.h:47) >> ==5467== by 0x3EF68B: lld::freeArena() (Memory.cpp:21) >> ==5467== by 0x4B3D41: lld::elf::link(llvm::ArrayRef<char const*>, >> bool, llvm::raw_ostream&) (Driver.cpp:101) >> ==5467== by 0x39FC06: ZigLLDLink (zig_llvm.cpp:840) >> ==5467== by 0x38DE74: link.resume (link.zig:84) >> ==5467== by 0x2475EB: Loop_workerRun (loop.zig:511) >> ==5467== by 0x2495EC: MainFuncs_posixThreadMain (index.zig:2680) >> ==5467== Block was alloc'd at >> ==5467== at 0x4C2D1AF: operator new(unsigned long) (in >> /nix/store/j7sf6f2i9gws7c5mn8jyi8wc08ab1zbp-valgrind-3.13.0/ >> lib/valgrind/vgpreload_memcheck-amd64-linux.so) >> ==5467== by 0x4CD027: llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::allocateBuckets(unsigned int) (DenseMap.h:794) >> ==5467== by 0x524C18: llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::grow(unsigned int) (DenseMap.h:732) >> ==5467== by 0x524077: llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::grow(unsigned int) (DenseMap.h:502) >> ==5467== by 0x522E54: llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>* llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::InsertIntoBucketImpl<llvm::CachedHashStringRef>(llvm::CachedHashStringRef >> const&, llvm::CachedHashStringRef const&, llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>*) (DenseMap.h:546) >> ==5467== by 0x521258: llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >> int>* llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::InsertIntoBucket<llvm::CachedHashStringRef, >> int>(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*, >> llvm::CachedHashStringRef&&, int&&) (DenseMap.h:512) >> ==5467== by 0x51DEC9: std::pair<llvm::DenseMapIterator<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>, false>, >> bool> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::try_emplace<int>(llvm::CachedHashStringRef&&, int&&) (DenseMap.h:215) >> ==5467== by 0x51C52F: llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, >> int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >> llvm::CachedHashStringRef, int, llvm::DenseMapInfo<llvm::CachedHashStringRef>, >> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >> >::insert(std::pair<llvm::CachedHashStringRef, int>&&) (DenseMap.h:199) >> ==5467== by 0x516836: lld::elf::SymbolTable::insert(llvm::StringRef) >> (SymbolTable.cpp:225) >> ==5467== by 0x516A21: lld::elf::SymbolTable::insert(llvm::StringRef, >> unsigned char, unsigned char, bool, lld::elf::InputFile*) >> (SymbolTable.cpp:260) >> ==5467== by 0x51B88C: void lld::elf::SymbolTable::addShar >> ed<llvm::object::ELFType<(llvm::support::endianness)1, true> >> >(llvm::StringRef, lld::elf::SharedFile<llvm::obj >> ect::ELFType<(llvm::support::endianness)1, true> >&, >> llvm::object::ELFType<(llvm::support::endianness)1, true>::Sym const&, >> unsigned int, unsigned int) (SymbolTable.cpp:483) >> ==5467== by 0x4E762A: lld::elf::SharedFile<llvm::obj >> ect::ELFType<(llvm::support::endianness)1, true> >::parseRest() >> (InputFiles.cpp:895) >> >> > > _______________________________________________ > LLVM Developers mailing list > llvm-dev at lists.llvm.org > http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev > >-------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.llvm.org/pipermail/llvm-dev/attachments/20180725/b189099d/attachment-0001.html>
Dave Bozier via llvm-dev
2018-Jul-25 11:08 UTC
[llvm-dev] are the LLD libraries thread safe?
Are there any plans to make LLD thread safe in the future? This would be a useful feature for us. Cheers, Dave On Wed, Jul 25, 2018 at 10:12 AM James Henderson via llvm-dev < llvm-dev at lists.llvm.org> wrote:> Hi Andrew, > > LLD relies on various bits of global state which are manipulated during > the link, so I wouldn't expect it to be thread safe at that level, although > it does attempt to reset that global state at the start of each call to > link(), so it should be callable sequentially. > > Regards, > > James > > On 25 July 2018 at 02:37, Andrew Kelley via llvm-dev < > llvm-dev at lists.llvm.org> wrote: > >> Follow-up - >> >> Putting a global lock on running LLD indeed solved the issue, so I'm >> guessing that the LLD libraries do not in fact support being called from >> multiple threads simultaneously. >> >> >> On Tue, Jul 24, 2018 at 9:10 PM, Andrew Kelley <superjoe30 at gmail.com> >> wrote: >> >>> E.g. Is it intended to be allowed to call lld::elf::link in 2 different >>> threads at the same time? >>> >>> Follows is an example Valgrind error I ran into when doing the above. >>> >>> I'll try putting a global resource lock on invoking LLD and see if it >>> solves the problem. >>> >>> ==5467== Invalid write of size 8 >>> ==5467== at 0x525509: >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::moveFromOldBuckets(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >>> int>*, llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >>> (DenseMap.h:375) >>> ==5467== by 0x524C7F: llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>> int) (DenseMap.h:739) >>> ==5467== by 0x524077: >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>> int) (DenseMap.h:502) >>> ==5467== by 0x522E54: >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::InsertIntoBucketImpl<llvm::CachedHashStringRef>(llvm::CachedHashStringRef >>> const&, llvm::CachedHashStringRef const&, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >>> (DenseMap.h:546) >>> ==5467== by 0x521258: >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::InsertIntoBucket<llvm::CachedHashStringRef, >>> int>(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*, >>> llvm::CachedHashStringRef&&, int&&) (DenseMap.h:512) >>> ==5467== by 0x51DEC9: >>> std::pair<llvm::DenseMapIterator<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>, false>, bool> >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::try_emplace<int>(llvm::CachedHashStringRef&&, int&&) (DenseMap.h:215) >>> ==5467== by 0x51C52F: >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::insert(std::pair<llvm::CachedHashStringRef, int>&&) (DenseMap.h:199) >>> ==5467== by 0x516836: lld::elf::SymbolTable::insert(llvm::StringRef) >>> (SymbolTable.cpp:225) >>> ==5467== by 0x516A21: lld::elf::SymbolTable::insert(llvm::StringRef, >>> unsigned char, unsigned char, bool, lld::elf::InputFile*) >>> (SymbolTable.cpp:260) >>> ==5467== by 0x51B88C: void >>> lld::elf::SymbolTable::addShared<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >(llvm::StringRef, >>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >&, llvm::object::ELFType<(llvm::support::endianness)1, true>::Sym >>> const&, unsigned int, unsigned int) (SymbolTable.cpp:483) >>> ==5467== by 0x4E762A: >>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >::parseRest() (InputFiles.cpp:895) >>> ==5467== by 0x51F69F: void >>> lld::elf::SymbolTable::addFile<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >(lld::elf::InputFile*) (SymbolTable.cpp:99) >>> ==5467== Address 0xc6c7e80 is 53,568 bytes inside a block of size >>> 98,304 free'd >>> ==5467== at 0x4C2E26B: operator delete(void*) (in >>> /nix/store/j7sf6f2i9gws7c5mn8jyi8wc08ab1zbp-valgrind-3.13.0/lib/valgrind/vgpreload_memcheck-amd64-linux.so) >>> ==5467== by 0x4C714C: llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::~DenseMap() >>> (DenseMap.h:681) >>> ==5467== by 0x4C9C7B: lld::elf::SymbolTable::~SymbolTable() >>> (SymbolTable.h:36) >>> ==5467== by 0x4C9BF1: >>> llvm::SpecificBumpPtrAllocator<lld::elf::SymbolTable>::DestroyAll()::{lambda(char*, >>> char*)#1}::operator()(char*, char*) const (Allocator.h:410) >>> ==5467== by 0x4C9D77: >>> llvm::SpecificBumpPtrAllocator<lld::elf::SymbolTable>::DestroyAll() >>> (Allocator.h:421) >>> ==5467== by 0x4D218F: >>> lld::SpecificAlloc<lld::elf::SymbolTable>::reset() (Memory.h:47) >>> ==5467== by 0x3EF68B: lld::freeArena() (Memory.cpp:21) >>> ==5467== by 0x4B3D41: lld::elf::link(llvm::ArrayRef<char const*>, >>> bool, llvm::raw_ostream&) (Driver.cpp:101) >>> ==5467== by 0x39FC06: ZigLLDLink (zig_llvm.cpp:840) >>> ==5467== by 0x38DE74: link.resume (link.zig:84) >>> ==5467== by 0x2475EB: Loop_workerRun (loop.zig:511) >>> ==5467== by 0x2495EC: MainFuncs_posixThreadMain (index.zig:2680) >>> ==5467== Block was alloc'd at >>> ==5467== at 0x4C2D1AF: operator new(unsigned long) (in >>> /nix/store/j7sf6f2i9gws7c5mn8jyi8wc08ab1zbp-valgrind-3.13.0/lib/valgrind/vgpreload_memcheck-amd64-linux.so) >>> ==5467== by 0x4CD027: llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::allocateBuckets(unsigned int) (DenseMap.h:794) >>> ==5467== by 0x524C18: llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>> int) (DenseMap.h:732) >>> ==5467== by 0x524077: >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>> int) (DenseMap.h:502) >>> ==5467== by 0x522E54: >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::InsertIntoBucketImpl<llvm::CachedHashStringRef>(llvm::CachedHashStringRef >>> const&, llvm::CachedHashStringRef const&, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >>> (DenseMap.h:546) >>> ==5467== by 0x521258: >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::InsertIntoBucket<llvm::CachedHashStringRef, >>> int>(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*, >>> llvm::CachedHashStringRef&&, int&&) (DenseMap.h:512) >>> ==5467== by 0x51DEC9: >>> std::pair<llvm::DenseMapIterator<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>, false>, bool> >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::try_emplace<int>(llvm::CachedHashStringRef&&, int&&) (DenseMap.h:215) >>> ==5467== by 0x51C52F: >>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>> llvm::CachedHashStringRef, int, >>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>> >::insert(std::pair<llvm::CachedHashStringRef, int>&&) (DenseMap.h:199) >>> ==5467== by 0x516836: lld::elf::SymbolTable::insert(llvm::StringRef) >>> (SymbolTable.cpp:225) >>> ==5467== by 0x516A21: lld::elf::SymbolTable::insert(llvm::StringRef, >>> unsigned char, unsigned char, bool, lld::elf::InputFile*) >>> (SymbolTable.cpp:260) >>> ==5467== by 0x51B88C: void >>> lld::elf::SymbolTable::addShared<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >(llvm::StringRef, >>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >&, llvm::object::ELFType<(llvm::support::endianness)1, true>::Sym >>> const&, unsigned int, unsigned int) (SymbolTable.cpp:483) >>> ==5467== by 0x4E762A: >>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>> true> >::parseRest() (InputFiles.cpp:895) >>> >>> >> >> _______________________________________________ >> LLVM Developers mailing list >> llvm-dev at lists.llvm.org >> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev >> >> > _______________________________________________ > LLVM Developers mailing list > llvm-dev at lists.llvm.org > http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev >-------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.llvm.org/pipermail/llvm-dev/attachments/20180725/a6fb90ba/attachment.html>
Rui Ueyama via llvm-dev
2018-Aug-22 07:46 UTC
[llvm-dev] are the LLD libraries thread safe?
It is unlikely due to the complexity of making everything thread-safe, but you can probably just invoke lld as a new process or do fork() to run multiple instances of lld concurrently. On Wed, Jul 25, 2018 at 8:09 PM Dave Bozier via llvm-dev < llvm-dev at lists.llvm.org> wrote:> Are there any plans to make LLD thread safe in the future? This would be a > useful feature for us. > > Cheers, > > Dave > > On Wed, Jul 25, 2018 at 10:12 AM James Henderson via llvm-dev < > llvm-dev at lists.llvm.org> wrote: > >> Hi Andrew, >> >> LLD relies on various bits of global state which are manipulated during >> the link, so I wouldn't expect it to be thread safe at that level, although >> it does attempt to reset that global state at the start of each call to >> link(), so it should be callable sequentially. >> >> Regards, >> >> James >> >> On 25 July 2018 at 02:37, Andrew Kelley via llvm-dev < >> llvm-dev at lists.llvm.org> wrote: >> >>> Follow-up - >>> >>> Putting a global lock on running LLD indeed solved the issue, so I'm >>> guessing that the LLD libraries do not in fact support being called from >>> multiple threads simultaneously. >>> >>> >>> On Tue, Jul 24, 2018 at 9:10 PM, Andrew Kelley <superjoe30 at gmail.com> >>> wrote: >>> >>>> E.g. Is it intended to be allowed to call lld::elf::link in 2 different >>>> threads at the same time? >>>> >>>> Follows is an example Valgrind error I ran into when doing the above. >>>> >>>> I'll try putting a global resource lock on invoking LLD and see if it >>>> solves the problem. >>>> >>>> ==5467== Invalid write of size 8 >>>> ==5467== at 0x525509: >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::moveFromOldBuckets(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, >>>> int>*, llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >>>> (DenseMap.h:375) >>>> ==5467== by 0x524C7F: llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>>> int) (DenseMap.h:739) >>>> ==5467== by 0x524077: >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>>> int) (DenseMap.h:502) >>>> ==5467== by 0x522E54: >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::InsertIntoBucketImpl<llvm::CachedHashStringRef>(llvm::CachedHashStringRef >>>> const&, llvm::CachedHashStringRef const&, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >>>> (DenseMap.h:546) >>>> ==5467== by 0x521258: >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::InsertIntoBucket<llvm::CachedHashStringRef, >>>> int>(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*, >>>> llvm::CachedHashStringRef&&, int&&) (DenseMap.h:512) >>>> ==5467== by 0x51DEC9: >>>> std::pair<llvm::DenseMapIterator<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>, false>, bool> >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::try_emplace<int>(llvm::CachedHashStringRef&&, int&&) (DenseMap.h:215) >>>> ==5467== by 0x51C52F: >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::insert(std::pair<llvm::CachedHashStringRef, int>&&) (DenseMap.h:199) >>>> ==5467== by 0x516836: lld::elf::SymbolTable::insert(llvm::StringRef) >>>> (SymbolTable.cpp:225) >>>> ==5467== by 0x516A21: lld::elf::SymbolTable::insert(llvm::StringRef, >>>> unsigned char, unsigned char, bool, lld::elf::InputFile*) >>>> (SymbolTable.cpp:260) >>>> ==5467== by 0x51B88C: void >>>> lld::elf::SymbolTable::addShared<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >(llvm::StringRef, >>>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >&, llvm::object::ELFType<(llvm::support::endianness)1, true>::Sym >>>> const&, unsigned int, unsigned int) (SymbolTable.cpp:483) >>>> ==5467== by 0x4E762A: >>>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >::parseRest() (InputFiles.cpp:895) >>>> ==5467== by 0x51F69F: void >>>> lld::elf::SymbolTable::addFile<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >(lld::elf::InputFile*) (SymbolTable.cpp:99) >>>> ==5467== Address 0xc6c7e80 is 53,568 bytes inside a block of size >>>> 98,304 free'd >>>> ==5467== at 0x4C2E26B: operator delete(void*) (in >>>> /nix/store/j7sf6f2i9gws7c5mn8jyi8wc08ab1zbp-valgrind-3.13.0/lib/valgrind/vgpreload_memcheck-amd64-linux.so) >>>> ==5467== by 0x4C714C: llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::~DenseMap() >>>> (DenseMap.h:681) >>>> ==5467== by 0x4C9C7B: lld::elf::SymbolTable::~SymbolTable() >>>> (SymbolTable.h:36) >>>> ==5467== by 0x4C9BF1: >>>> llvm::SpecificBumpPtrAllocator<lld::elf::SymbolTable>::DestroyAll()::{lambda(char*, >>>> char*)#1}::operator()(char*, char*) const (Allocator.h:410) >>>> ==5467== by 0x4C9D77: >>>> llvm::SpecificBumpPtrAllocator<lld::elf::SymbolTable>::DestroyAll() >>>> (Allocator.h:421) >>>> ==5467== by 0x4D218F: >>>> lld::SpecificAlloc<lld::elf::SymbolTable>::reset() (Memory.h:47) >>>> ==5467== by 0x3EF68B: lld::freeArena() (Memory.cpp:21) >>>> ==5467== by 0x4B3D41: lld::elf::link(llvm::ArrayRef<char const*>, >>>> bool, llvm::raw_ostream&) (Driver.cpp:101) >>>> ==5467== by 0x39FC06: ZigLLDLink (zig_llvm.cpp:840) >>>> ==5467== by 0x38DE74: link.resume (link.zig:84) >>>> ==5467== by 0x2475EB: Loop_workerRun (loop.zig:511) >>>> ==5467== by 0x2495EC: MainFuncs_posixThreadMain (index.zig:2680) >>>> ==5467== Block was alloc'd at >>>> ==5467== at 0x4C2D1AF: operator new(unsigned long) (in >>>> /nix/store/j7sf6f2i9gws7c5mn8jyi8wc08ab1zbp-valgrind-3.13.0/lib/valgrind/vgpreload_memcheck-amd64-linux.so) >>>> ==5467== by 0x4CD027: llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::allocateBuckets(unsigned int) (DenseMap.h:794) >>>> ==5467== by 0x524C18: llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>>> int) (DenseMap.h:732) >>>> ==5467== by 0x524077: >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >::grow(unsigned >>>> int) (DenseMap.h:502) >>>> ==5467== by 0x522E54: >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::InsertIntoBucketImpl<llvm::CachedHashStringRef>(llvm::CachedHashStringRef >>>> const&, llvm::CachedHashStringRef const&, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*) >>>> (DenseMap.h:546) >>>> ==5467== by 0x521258: >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>* >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::InsertIntoBucket<llvm::CachedHashStringRef, >>>> int>(llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>*, >>>> llvm::CachedHashStringRef&&, int&&) (DenseMap.h:512) >>>> ==5467== by 0x51DEC9: >>>> std::pair<llvm::DenseMapIterator<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int>, false>, bool> >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::try_emplace<int>(llvm::CachedHashStringRef&&, int&&) (DenseMap.h:215) >>>> ==5467== by 0x51C52F: >>>> llvm::DenseMapBase<llvm::DenseMap<llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >, >>>> llvm::CachedHashStringRef, int, >>>> llvm::DenseMapInfo<llvm::CachedHashStringRef>, >>>> llvm::detail::DenseMapPair<llvm::CachedHashStringRef, int> >>>> >::insert(std::pair<llvm::CachedHashStringRef, int>&&) (DenseMap.h:199) >>>> ==5467== by 0x516836: lld::elf::SymbolTable::insert(llvm::StringRef) >>>> (SymbolTable.cpp:225) >>>> ==5467== by 0x516A21: lld::elf::SymbolTable::insert(llvm::StringRef, >>>> unsigned char, unsigned char, bool, lld::elf::InputFile*) >>>> (SymbolTable.cpp:260) >>>> ==5467== by 0x51B88C: void >>>> lld::elf::SymbolTable::addShared<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >(llvm::StringRef, >>>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >&, llvm::object::ELFType<(llvm::support::endianness)1, true>::Sym >>>> const&, unsigned int, unsigned int) (SymbolTable.cpp:483) >>>> ==5467== by 0x4E762A: >>>> lld::elf::SharedFile<llvm::object::ELFType<(llvm::support::endianness)1, >>>> true> >::parseRest() (InputFiles.cpp:895) >>>> >>>> >>> >>> _______________________________________________ >>> LLVM Developers mailing list >>> llvm-dev at lists.llvm.org >>> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev >>> >>> >> _______________________________________________ >> LLVM Developers mailing list >> llvm-dev at lists.llvm.org >> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev >> > _______________________________________________ > LLVM Developers mailing list > llvm-dev at lists.llvm.org > http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-dev >-------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.llvm.org/pipermail/llvm-dev/attachments/20180822/9510470f/attachment-0001.html>