//===--- iwyu_include_picker.cc - map to canonical #includes for iwyu -----===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "iwyu_include_picker.h" #include // for find #include // for size_t // TODO(wan): make sure IWYU doesn't suggest . #include // for find // not hash_map: it's not as portable and needs hash. #include // for map, map<>::mapped_type, etc #include #include // for string, basic_string, etc #include // for error_code #include // for pair, make_pair #include // for vector, vector<>::iterator #include "iwyu_location_util.h" #include "iwyu_path_util.h" #include "iwyu_port.h" #include "iwyu_regex.h" #include "iwyu_stl_util.h" #include "iwyu_string_util.h" #include "iwyu_verrs.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/YAMLParser.h" #include "clang/Basic/FileManager.h" using std::find; using std::make_pair; using std::map; using std::pair; using std::string; using std::unique_ptr; using std::vector; using llvm::MemoryBuffer; using llvm::SourceMgr; using llvm::yaml::KeyValueNode; using llvm::yaml::MappingNode; using llvm::yaml::Node; using llvm::yaml::ScalarNode; using llvm::yaml::SequenceNode; using llvm::yaml::Stream; using llvm::yaml::document_iterator; namespace include_what_you_use { // If we map from A to B, it means that every time we need a // symbol from A, we can also get it from B. Another way // to think about it is that map_to "re-exports" all the // symbols from map_from. struct IncludeMapEntry { // A POD so we can make the input static const char* map_from; // A quoted-include or a symbol name IncludeVisibility from_visibility; const char* map_to; // A quoted-include IncludeVisibility to_visibility; }; namespace { // Listed below are all IWYU's native symbol and include mappings, // loosely based on GCC 4.4's libc and libstdc++. // Symbol -> include mappings for GNU libc const IncludeMapEntry libc_symbol_map[] = { // For library symbols that can be defined in more than one header // file, maps from symbol-name to legitimate header files. // This list was generated via // grep -R '__.*_defined' /usr/include | perl -nle 'm,/usr/include/([^:]*):#\s*\S+ __(.*)_defined, and print qq@ { "$2", kPublic, "<$1>", kPublic },@' | sort -u // I ignored all entries that only appeared once on the list (eg uint32_t). // I then added in NULL, which according to [diff.null] C.2.2.3, can // be defined in , , , , // , , or . We also allow their C // equivalents. // In each case, I ordered them so was first, if it was // an option for this type. That's the preferred #include all else // equal. The visibility on the symbol-name is ignored; by convention // we always set it to kPrivate. { "aiocb", kPrivate, "", kPublic }, { "blkcnt_t", kPrivate, "", kPublic }, { "blkcnt_t", kPrivate, "", kPublic }, { "blksize_t", kPrivate, "", kPublic }, { "blksize_t", kPrivate, "", kPublic }, { "cc_t", kPrivate, "", kPublic }, { "clock_t", kPrivate, "", kPublic }, { "clock_t", kPrivate, "", kPublic }, { "clock_t", kPrivate, "", kPublic }, { "clockid_t", kPrivate, "", kPublic }, { "clockid_t", kPrivate, "", kPublic }, { "daddr_t", kPrivate, "", kPublic }, { "daddr_t", kPrivate, "", kPublic }, { "dev_t", kPrivate, "", kPublic }, { "dev_t", kPrivate, "", kPublic }, { "div_t", kPrivate, "", kPublic }, { "double_t", kPrivate, "", kPublic }, { "error_t", kPrivate, "", kPublic }, { "error_t", kPrivate, "", kPublic }, { "error_t", kPrivate, "", kPublic }, { "fd_set", kPrivate, "", kPublic }, { "fd_set", kPrivate, "", kPublic }, { "fenv_t", kPrivate, "", kPublic }, { "fexcept_t", kPrivate, "", kPublic }, { "FILE", kPrivate, "", kPublic }, { "FILE", kPrivate, "", kPublic }, { "float_t", kPrivate, "", kPublic }, { "fsblkcnt_t", kPrivate, "", kPublic }, { "fsblkcnt_t", kPrivate, "", kPublic }, { "fsfilcnt_t", kPrivate, "", kPublic }, { "fsfilcnt_t", kPrivate, "", kPublic }, { "getopt", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "gid_t", kPrivate, "", kPublic }, { "htonl", kPrivate, "", kPublic }, { "htons", kPrivate, "", kPublic }, { "id_t", kPrivate, "", kPublic }, { "id_t", kPrivate, "", kPublic }, { "imaxdiv_t", kPrivate, "", kPublic }, { "intmax_t", kPrivate, "", kPublic }, { "uintmax_t", kPrivate, "", kPublic }, { "ino64_t", kPrivate, "", kPublic }, { "ino64_t", kPrivate, "", kPublic }, { "ino_t", kPrivate, "", kPublic }, { "ino_t", kPrivate, "", kPublic }, { "ino_t", kPrivate, "", kPublic }, { "int8_t", kPrivate, "", kPublic }, { "int16_t", kPrivate, "", kPublic }, { "int32_t", kPrivate, "", kPublic }, { "int64_t", kPrivate, "", kPublic }, { "uint8_t", kPrivate, "", kPublic }, { "uint16_t", kPrivate, "", kPublic }, { "uint32_t", kPrivate, "", kPublic }, { "uint64_t", kPrivate, "", kPublic }, { "intptr_t", kPrivate, "", kPublic }, { "uintptr_t", kPrivate, "", kPublic }, { "iovec", kPrivate, "", kPublic }, { "iovec", kPrivate, "", kPublic }, { "itimerspec", kPrivate, "", kPublic }, { "itimerspec", kPrivate, "", kPublic }, { "key_t", kPrivate, "", kPublic }, { "key_t", kPrivate, "", kPublic }, { "lconv", kPrivate, "", kPublic }, { "ldiv_t", kPrivate, "", kPublic }, { "lldiv_t", kPrivate, "", kPublic }, { "max_align_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "mode_t", kPrivate, "", kPublic }, { "nlink_t", kPrivate, "", kPublic }, { "nlink_t", kPrivate, "", kPublic }, { "ntohl", kPrivate, "", kPublic }, { "ntohs", kPrivate, "", kPublic }, { "off64_t", kPrivate, "", kPublic }, { "off64_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "off_t", kPrivate, "", kPublic }, { "optarg", kPrivate, "", kPublic }, { "opterr", kPrivate, "", kPublic }, { "optind", kPrivate, "", kPublic }, { "optopt", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "pid_t", kPrivate, "", kPublic }, { "ptrdiff_t", kPrivate, "", kPublic }, { "regex_t", kPrivate, "", kPublic }, { "regmatch_t", kPrivate, "", kPublic }, { "regoff_t", kPrivate, "", kPublic }, { "sigevent", kPrivate, "", kPublic }, { "sigevent", kPrivate, "", kPublic }, { "sigevent", kPrivate, "", kPublic }, { "sigevent", kPrivate, "", kPublic }, { "siginfo_t", kPrivate, "", kPublic }, { "siginfo_t", kPrivate, "", kPublic }, { "sigset_t", kPrivate, "", kPublic }, { "sigset_t", kPrivate, "", kPublic }, { "sigset_t", kPrivate, "", kPublic }, { "sigval", kPrivate, "", kPublic }, { "sockaddr", kPrivate, "", kPublic }, { "socklen_t", kPrivate, "", kPublic }, { "socklen_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "ssize_t", kPrivate, "", kPublic }, { "stat", kPrivate, "", kPublic }, { "stat", kPrivate, "", kPublic }, { "suseconds_t", kPrivate, "", kPublic }, { "suseconds_t", kPrivate, "", kPublic }, { "suseconds_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "time_t", kPrivate, "", kPublic }, { "timer_t", kPrivate, "", kPublic }, { "timer_t", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timespec", kPrivate, "", kPublic }, { "timeval", kPrivate, "", kPublic }, { "timeval", kPrivate, "", kPublic }, { "timeval", kPrivate, "", kPublic }, { "timeval", kPrivate, "", kPublic }, { "tm", kPrivate, "", kPublic }, { "u_char", kPrivate, "", kPublic }, { "u_char", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "uid_t", kPrivate, "", kPublic }, { "useconds_t", kPrivate, "", kPublic }, { "useconds_t", kPrivate, "", kPublic }, { "wchar_t", kPrivate, "", kPublic }, { "wchar_t", kPrivate, "", kPublic }, // It is unspecified if the cname headers provide ::size_t. // is the one header which defines NULL but not size_t. { "size_t", kPrivate, "", kPublic }, // 'canonical' location for size_t { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, { "size_t", kPrivate, "", kPublic }, // Macros that can be defined in more than one file, don't have the // same __foo_defined guard that other types do, so the grep above // doesn't discover them. Until I figure out a better way, I just // add them in by hand as I discover them. { "EOF", kPrivate, "", kPublic }, { "EOF", kPrivate, "", kPublic }, { "FILE", kPrivate, "", kPublic }, { "MAP_POPULATE", kPrivate, "", kPublic }, { "MAP_POPULATE", kPrivate, "", kPublic }, { "MAP_STACK", kPrivate, "", kPublic }, { "MAP_STACK", kPrivate, "", kPublic }, { "MAXHOSTNAMELEN", kPrivate, "", kPublic }, { "MAXHOSTNAMELEN", kPrivate, "", kPublic }, { "SIGABRT", kPrivate, "", kPublic }, { "SIGCHLD", kPrivate, "", kPublic }, { "SIGCHLD", kPrivate, "", kPublic }, { "va_list", kPrivate, "", kPublic }, { "va_list", kPrivate, "", kPublic }, { "va_list", kPrivate, "", kPublic }, // These are symbols that could be defined in either stdlib.h or // malloc.h, but we always want the stdlib location. { "malloc", kPrivate, "", kPublic }, { "calloc", kPrivate, "", kPublic }, { "realloc", kPrivate, "", kPublic }, { "free", kPrivate, "", kPublic }, // Entries for NULL { "NULL", kPrivate, "", kPublic }, // 'canonical' location for NULL { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, { "NULL", kPrivate, "", kPublic }, }; // Symbol -> include mappings for GNU libstdc++ const IncludeMapEntry libstdcpp_symbol_map[] = { // Kludge time: almost all STL types take an allocator, but they // almost always use the default value. Usually we detect that // and don't try to do IWYU, but sometimes it passes through. // For instance, when adding two strings, we end up calling // template<_CharT,_Traits,_Alloc> ... operator+( // basic_string<_CharT,_Traits,_Alloc>, ...) // These look like normal template args to us, so we see they're // used and declare an iwyu dependency, even though we don't need // to #include the traits or alloc type ourselves. The surest way // to deal with this is to just say that everyone provides // std::allocator. We can add more here at need. { "std::allocator", kPrivate, "", kPublic }, { "std::allocator", kPrivate, "", kPublic }, { "std::allocator", kPrivate, "", kPublic }, { "std::allocator", kPrivate, "", kPublic }, { "std::allocator", kPrivate, "", kPublic }, // A similar kludge for std::char_traits. basic_string, // basic_ostream and basic_istream have this as a default template // argument, and sometimes it bleeds through when clang desugars the // string/ostream/istream type. { "std::char_traits", kPrivate, "", kPublic }, { "std::char_traits", kPrivate, "", kPublic }, { "std::char_traits", kPrivate, "", kPublic }, { "std::size_t", kPrivate, "", kPublic }, // 'canonical' location for std::size_t { "std::size_t", kPrivate, "", kPublic }, { "std::size_t", kPrivate, "", kPublic }, { "std::size_t", kPrivate, "", kPublic }, { "std::size_t", kPrivate, "", kPublic }, { "std::size_t", kPrivate, "", kPublic }, { "std::size_t", kPrivate, "", kPublic }, }; const IncludeMapEntry libc_include_map[] = { // Private -> public include mappings for GNU libc // ( cd /usr/include && grep '^ *# *include' {sys/,net/,}* | perl -nle 'm/^([^:]+).*<([^>]+)>/ && print qq@ { "<$2>", kPrivate, "<$1>", kPublic },@' | grep bits/ | sort ) // When I saw more than one mapping for these, I typically picked // what I thought was the "best" one. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // Sometimes libc tells you what mapping to do via an '#error': // # error "Never use directly; include instead." // or // # error "Never include directly; use instead." // ( cd /usr/include && grep -R '^ *# *error "Never use\|include' * | perl -nle 'm/<([^>]+).*directly.*<([^>]+)/ && print qq@ { "<$1>", kPrivate, "<$2>", kPublic },@' | sort ) { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // Top-level #includes that just forward to another file: // $ for i in /usr/include/*; do [ -f $i ] && [ `wc -l < $i` = 1 ] && echo $i; done // (poll.h, syscall.h, syslog.h, ustat.h, wait.h). // For each file, I looked at the list of canonical header files -- // http://www.opengroup.org/onlinepubs/9699919799/idx/head.html -- // to decide which of the two files is canonical. If neither is // on the POSIX.1 1998 list, I just choose the top-level one. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // These are all files in bits/ that delegate to asm/ and linux/ to // do all (or lots) of the work. Note these are private->private. // $ for i in /usr/include/bits/*; do for dir in asm linux; do grep -H -e $dir/`basename $i` $i; done; done { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, // Some asm files have 32- and 64-bit variants: // $ ls /usr/include/asm/*_{32,64}.h { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, // I don't know what grep would have found these. I found them // via user report. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // PATH_MAX { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // Exports guaranteed by the C standard { "", kPublic, "", kPublic }, }; const IncludeMapEntry stdlib_c_include_map[] = { // Allow the C++ wrappers around C files. Without these mappings, // if you #include , iwyu will tell you to replace it with // , which is where the symbols are actually defined. We // inhibit that behavior to keep the alone. Note this is a // public-to-public mapping: we don't want to *replace* // with , we just want to avoid suggesting changing // back to . (If you *did* want to replace // assert.h with cassert, you'd change it to a public->private // mapping.) Here is how I identified the files to map: // $ for i in /usr/include/c++/4.4/c* ; do ls /usr/include/`basename $i | cut -b2-`.h /usr/lib/gcc/*/4.4/include/`basename $i | cut -b2-`.h 2>/dev/null ; done // // These headers are defined in [headers.cpp.c]. // https://github.com/cplusplus/draft/blob/c+%2B20/source/lib-intro.tex // // $ curl -s -N https://raw.githubusercontent.com/cplusplus/draft/c%2B%2B20/source/lib-intro.tex | sed -n '/begin{multicolfloattable}.*{headers.cpp.c}/,/end{multicolfloattable}/p' lib-intro.tex | grep tcode | perl -nle 'm/tcode{}/ && print qq@ { "<$1.h>", kPublic, "", kPublic },@' | sort { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, }; const char* stdlib_cpp_public_headers[] = { // These headers are defined in [headers.cpp]. // https://github.com/cplusplus/draft/blob/c+%2B20/source/lib-intro.tex // // $ curl -s -N https://raw.githubusercontent.com/cplusplus/draft/c%2B%2B20/source/lib-intro.tex | sed -n '/begin{multicolfloattable}.*{headers.cpp}/,/end{multicolfloattable}/p' lib-intro.tex | grep tcode | perl -nle 'm/tcode{(.*)}/ && print qq@ "$1",@' | sort "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", }; // Private -> public include mappings for GNU libstdc++ // // Note: make sure to sync this setting with gcc.stl.headers.imp // const IncludeMapEntry libstdcpp_include_map[] = { // cd /usr/include/c++/10 && grep -r headername | perl -nle 'm/^([^:]+).*@headername\{([^,]*)\}/ && print qq@ { "<$1>", kPrivate, "<$2>", kPublic },@' | sort -u { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // cd /usr/include/x86_64-linux-gnu/c++/10 && grep -r headername | perl -nle 'm/^([^:]+).*@headername\{([^,]*)\}/ && print qq@ { "<$1>", kPrivate, "<$2>", kPublic },@' | sort -u { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // ( cd /usr/crosstool/v12/gcc-4.3.1-glibc-2.3.6-grte/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/include/c++/4.3.1 && grep '^ *# *include' {ext/,tr1/,}* | perl -nle 'm/^([^:]+).*<([^>]+)>/ && print qq@ { "<$2>", kPrivate, "<$1>", kPublic },@' | grep -e bits/ -e tr1_impl/ | sort -u) // I removed a lot of 'meaningless' dependencies -- for instance, // #includes , but if someone is // using strings, isn't enough to satisfy iwyu. // We may need to add other dirs in future versions of gcc. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // Hash and hashtable-based containers. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // All .tcc files are gcc internal-include files. We get them from // ( cd /usr/crosstool/v12/gcc-4.3.1-glibc-2.3.6-grte/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/include/c++/4.3.1 && grep -R '^ *# *include.*tcc' * | perl -nle 'm/^([^:]+).*[<"]([^>"]+)[>"]/ && print qq@ { "<$2>", kPrivate, "<$1>", kPublic },@' | sort ) // I had to manually edit some of the entries to say the map-to is private. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPrivate }, // Some bits->bits #includes: A few files in bits re-export // symbols from other files in bits. // ( cd /usr/crosstool/v12/gcc-4.3.1-glibc-2.3.6-grte/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/include/c++/4.3.1 && grep '^ *# *include.*bits/' bits/* | perl -nle 'm/^([^:]+).*<([^>]+)>/ && print qq@ { "<$2>", kPrivate, "<$1>", kPrivate },@' | grep bits/ | sort -u) // and carefully picked reasonable-looking results (algorithm // *uses* pair but doesn't *re-export* pair, for instance). { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, { "", kPrivate, "", kPrivate }, // I don't think we want to be having people move to 'backward/' // yet. (These hold deprecated STL classes that we still use // actively.) These are the ones that turned up in an analysis of { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // We have backward as part of the -I search path now, so have the // non-backwards-prefix version as well. { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, // (This one should perhaps be found automatically somehow.) { "", kPrivate, "", kPublic }, // The iostream .h files are confusing. Lots of private headers, // which are handled above, but we also have public headers // #including each other (eg #includes ). We // are pretty forgiving: if a user specifies any public header, we // generally don't require the others. // ( cd /usr/crosstool/v12/gcc-4.3.1-glibc-2.3.6-grte/x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/include/c++/4.3.1 && egrep '^ *# *include <(istream|ostream|iostream|fstream|sstream|streambuf|ios|iosfwd)>' *stream* ios | perl -nle 'm/^([^:]+).*[<"]([^>"]+)[>"]/ and print qq@ { "<$2>", kPublic, "<$1>", kPublic },@' | sort -u ) { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, { "", kPublic, "", kPublic }, // The location of exception_defines.h varies by GCC version. It should // never be included directly. { "", kPrivate, "", kPublic }, // post libstdc++-10 stuff which is not automatically caught by commands above { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, { "", kPrivate, "", kPublic }, }; // Returns true if str is a valid quoted filepath pattern (i.e. either // a quoted filepath or "@" followed by a regex for matching a quoted // filepath). bool IsQuotedFilepathPattern(const string& str) { return IsQuotedInclude(str) || StartsWith(str, "@"); } // Given a vector of nodes, augment each node with its children, as // defined by m: nodes[i] is replaced by nodes[i] + m[nodes[i]], // ignoring duplicates. The input vector is modified in place. void ExpandOnce(const IncludePicker::IncludeMap& m, vector* nodes) { vector nodes_and_children; set seen_nodes_and_children; for (const MappedInclude& node : *nodes) { // First insert the node itself, then all its kids. if (!ContainsKey(seen_nodes_and_children, node.quoted_include)) { nodes_and_children.push_back(node); seen_nodes_and_children.insert(node.quoted_include); } if (const vector* children = FindInMap(&m, node.quoted_include)) { for (const MappedInclude& child : *children) { if (!ContainsKey(seen_nodes_and_children, child.quoted_include)) { nodes_and_children.push_back(child); seen_nodes_and_children.insert(child.quoted_include); } } } } nodes->swap(nodes_and_children); // modify nodes in-place } enum TransitiveStatus { kUnused = 0, kCalculating, kDone }; // If the filename-map maps a.h to b.h, and also b.h to c.h, then // there's a transitive mapping of a.h to c.h. We want to add that // into the filepath map as well, to make lookups easier. We do this // by doing a depth-first search for a single mapping, recursing // whenever the value is itself a key in the map, and putting the // results in a vector of all values seen. // NOTE: This function updates values seen in filename_map, but // does not invalidate any filename_map iterators. void MakeNodeTransitive(IncludePicker::IncludeMap* filename_map, map* seen_nodes, vector* node_stack, // used for debugging const string& key) { // If we've already calculated this node's transitive closure, we're done. const TransitiveStatus status = (*seen_nodes)[key]; if (status == kCalculating) { // means there's a cycle in the mapping // Note that cycles in mappings are generally benign, the cycle detection // here is only necessary to protect the recursive algorithm from infinite // regress. We will still expand all reachable nodes in the graph to a // plain sequence representing the transitive closure. // The expanded mappings are only used for simple lookup, never followed // recursively (which could have necessitated preserving cycles and handling // them in that traversal too). // Log cycles at a high verbosity level to aid debugging. VERRS(8) << "Ignored cycle in include mappings: "; for (const string& node : *node_stack) VERRS(8) << node << " -> "; VERRS(8) << key << "\n"; return; } if (status == kDone) return; IncludePicker::IncludeMap::iterator node = filename_map->find(key); if (node == filename_map->end()) { (*seen_nodes)[key] = kDone; return; } // Keep track of node->second as we update it, to avoid duplicates. (*seen_nodes)[key] = kCalculating; for (const MappedInclude& child : node->second) { node_stack->push_back(child.quoted_include); MakeNodeTransitive(filename_map, seen_nodes, node_stack, child.quoted_include); node_stack->pop_back(); } (*seen_nodes)[key] = kDone; // Our transitive closure is just the union of the closure of our // children. This routine replaces our value with this closure, // by replacing each of our values with its values. Since our // values have already been made transitive, that is a closure. ExpandOnce(*filename_map, &node->second); } // Updates the values in filename_map based on its transitive mappings. void MakeMapTransitive(IncludePicker::IncludeMap* filename_map) { // Insert keys of filename_map here once we know their value is // the complete transitive closure. map seen_nodes; vector node_stack; for (const IncludePicker::IncludeMap::value_type& includes : *filename_map) MakeNodeTransitive(filename_map, &seen_nodes, &node_stack, includes.first); } // Get a scalar value from a YAML node. // Returns empty string if it's not of type ScalarNode. string GetScalarValue(Node* node) { ScalarNode* scalar = llvm::dyn_cast(node); if (scalar == nullptr) return string(); llvm::SmallString<8> storage; return scalar->getValue(storage).str(); } // Get a sequence value from a YAML node. // Returns empty vector if it's not of type SequenceNode. vector GetSequenceValue(Node* node) { vector result; SequenceNode* sequence = llvm::dyn_cast(node); if (sequence != nullptr) { for (Node& node : *sequence) { result.push_back(GetScalarValue(&node)); } } return result; } // If new_path doesn't already exist in search_path, makes a copy of search_path // and adds new_path to it. // Returns the original or extended search path. vector ExtendMappingFileSearchPath(const vector& search_path, const string& new_path) { CHECK_(IsAbsolutePath(new_path)); if (std::find(search_path.begin(), search_path.end(), new_path) == search_path.end()) { vector extended(search_path); extended.push_back(new_path); return extended; } return search_path; } // Scans search_path for existing files with filename. // If filename is absolute and exists, return it. // If filename is relative and exists based on cwd, return it in absolute form. // If filename is relative and doesn't exist, try to find it along search_path. // Returns an absolute filename if file is found, otherwise filename untouched. string FindFileInSearchPath(const vector& search_path, const string& filename) { if (llvm::sys::fs::exists(filename)) { // If the file exists, no matter if its path is relative or absolute, // return it in absolute form. return MakeAbsolutePath(filename); } else if (!IsAbsolutePath(filename)) { // If it's relative, scan search path. for (const string& base_path : search_path) { string candidate = MakeAbsolutePath(base_path, filename); if (llvm::sys::fs::exists(candidate)) { return candidate; } } } // This is proven not to exist, so handle the error when // we attempt to open it. return filename; } } // anonymous namespace MappedInclude::MappedInclude(const string& q, const string& p) : quoted_include(q) , path(p) { CHECK_(IsQuotedInclude(quoted_include)) << "Must be quoted include, was: " << quoted_include; } bool MappedInclude::HasAbsoluteQuotedInclude() const { if (!StartsWith(quoted_include, "\"") || quoted_include.size() < 2) { return false; } string path(quoted_include.begin() + 1, quoted_include.end() - 1); return IsAbsolutePath(path); } IncludePicker::IncludePicker(bool no_default_mappings, RegexDialect regex_dialect) : has_called_finalize_added_include_lines_(false), regex_dialect(regex_dialect) { if (!no_default_mappings) { AddDefaultMappings(); } } void IncludePicker::AddDefaultMappings() { AddSymbolMappings(libc_symbol_map, IWYU_ARRAYSIZE(libc_symbol_map)); AddSymbolMappings(libstdcpp_symbol_map, IWYU_ARRAYSIZE(libstdcpp_symbol_map)); AddIncludeMappings(libc_include_map, IWYU_ARRAYSIZE(libc_include_map)); AddIncludeMappings(stdlib_c_include_map, IWYU_ARRAYSIZE(stdlib_c_include_map)); AddIncludeMappings(libstdcpp_include_map, IWYU_ARRAYSIZE(libstdcpp_include_map)); AddPublicIncludes(stdlib_cpp_public_headers, IWYU_ARRAYSIZE(stdlib_cpp_public_headers)); } void IncludePicker::MarkVisibility(VisibilityMap* map, const string& key, IncludeVisibility visibility) { CHECK_(!has_called_finalize_added_include_lines_ && "Can't mutate anymore"); // insert() leaves any old value alone, and only inserts if the key is new. map->insert(make_pair(key, visibility)); CHECK_((*map)[key] == visibility) << " Same file seen with two different visibilities: " << key << " Old vis: " << (*map)[key] << " New vis: " << visibility; } // AddDirectInclude lets us use some hard-coded rules to add filepath // mappings at runtime. It includes, for instance, mappings from // 'project/internal/foo.h' to 'project/public/foo_public.h' in google // code (Google hides private headers in /internal/, much like glibc // hides them in /bits/.) void IncludePicker::AddDirectInclude(const string& includer_filepath, const string& includee_filepath, const string& quoted_include_as_written) { CHECK_(!has_called_finalize_added_include_lines_ && "Can't mutate anymore"); // Note: the includer may be a .cc file, which is unnecessary to add // to our map, but harmless. const string quoted_includer = ConvertToQuotedInclude(includer_filepath); const string quoted_includee = ConvertToQuotedInclude(includee_filepath); MappedInclude mapped_includer(quoted_includer, includer_filepath); quoted_includes_to_quoted_includers_[quoted_includee].insert(quoted_includer); const pair key(includer_filepath, includee_filepath); includer_and_includee_to_include_as_written_[key] = quoted_include_as_written; // Mark the clang fake-file "" as private, so we never try // to map anything to it. if (includer_filepath == "") MarkIncludeAsPrivate("\"\""); // Automatically mark files in foo/internal/bar as private, and map them. // Then say that everyone else in foo/.* is a friend, who is allowed to // include the otherwise-private header. const size_t internal_pos = quoted_includee.find("internal/"); if (internal_pos != string::npos && (internal_pos == 0 || quoted_includee[internal_pos - 1] == '/')) { MarkIncludeAsPrivate(quoted_includee); // The second argument here is a regex for matching a quoted // filepath. We get the opening quote from quoted_includee, and // the closing quote as part of the .*. AddFriendRegex(includee_filepath, quoted_includee.substr(0, internal_pos) + ".*"); VERRS(8) << "Adding dynamic mapping for internal/ header\n"; AddMapping(quoted_includee, mapped_includer); } // Automatically mark as private, and map to . if (StartsWith(quoted_includee, " VERRS(8) << "Adding dynamic mapping for header\n"; AddMapping(quoted_includee, MappedInclude(public_header)); } } void IncludePicker::AddMapping(const string& map_from, const MappedInclude& map_to) { VERRS(8) << "Adding mapping from " << map_from << " to " << map_to.quoted_include << "\n"; CHECK_(!has_called_finalize_added_include_lines_ && "Can't mutate anymore"); CHECK_(IsQuotedFilepathPattern(map_from) && "All map keys must be quoted filepaths or @ followed by regex"); filepath_include_map_[map_from].push_back(map_to); } void IncludePicker::AddIncludeMapping(const string& map_from, IncludeVisibility from_visibility, const MappedInclude& map_to, IncludeVisibility to_visibility) { AddMapping(map_from, map_to); MarkVisibility(&include_visibility_map_, map_from, from_visibility); MarkVisibility(&include_visibility_map_, map_to.quoted_include, to_visibility); } void IncludePicker::AddSymbolMapping(const string& map_from, const MappedInclude& map_to, IncludeVisibility to_visibility) { symbol_include_map_[map_from].push_back(map_to); MarkVisibility(&include_visibility_map_, map_to.quoted_include, to_visibility); } void IncludePicker::AddIncludeMappings(const IncludeMapEntry* entries, size_t count) { for (size_t i = 0; i < count; ++i) { const IncludeMapEntry& e = entries[i]; AddIncludeMapping(e.map_from, e.from_visibility, MappedInclude(e.map_to), e.to_visibility); } } void IncludePicker::AddSymbolMappings(const IncludeMapEntry* entries, size_t count) { for (size_t i = 0; i < count; ++i) { const IncludeMapEntry& e = entries[i]; AddSymbolMapping(e.map_from, MappedInclude(e.map_to), e.to_visibility); } } void IncludePicker::AddPublicIncludes(const char** includes, size_t count) { for (size_t i = 0; i < count; ++i) { const char* include = includes[i]; MarkVisibility(&include_visibility_map_, include, kPublic); } } void IncludePicker::MarkIncludeAsPrivate( const string& quoted_filepath_pattern) { CHECK_(!has_called_finalize_added_include_lines_ && "Can't mutate anymore"); CHECK_(IsQuotedFilepathPattern(quoted_filepath_pattern) && "MIAP takes a quoted filepath pattern"); MarkVisibility(&include_visibility_map_, quoted_filepath_pattern, kPrivate); } void IncludePicker::MarkPathAsPrivate(const string& path) { CHECK_(!has_called_finalize_added_include_lines_ && "Can't mutate anymore"); MarkVisibility(&path_visibility_map_, path, kPrivate); } void IncludePicker::AddFriendRegex(const string& includee_filepath, const string& quoted_friend_regex) { friend_to_headers_map_["@" + quoted_friend_regex].insert(includee_filepath); } namespace { // Given a map keyed by quoted filepath patterns, return a vector // containing the @-regexes among the keys. template vector ExtractKeysMarkedAsRegexes(const MapType& m) { vector regex_keys; for (const typename MapType::value_type& item : m) { if (StartsWith(item.first, "@")) regex_keys.push_back(item.first); } return regex_keys; } bool ContainsQuotedInclude( const vector& mapped_includes, const string& quoted_include) { for (const MappedInclude& mapped : mapped_includes) { if (mapped.quoted_include == quoted_include) { return true; } } return false; } } // anonymous namespace // Expands the regex keys in filepath_include_map_ and // friend_to_headers_map_ by matching them against all source files // seen by iwyu. For each include that matches the regex, we add it // to the map by copying the regex entry and replacing the key with // the seen #include. void IncludePicker::ExpandRegexes() { // First, get the regex keys. const vector filepath_include_map_regex_keys = ExtractKeysMarkedAsRegexes(filepath_include_map_); const vector friend_to_headers_map_regex_keys = ExtractKeysMarkedAsRegexes(friend_to_headers_map_); // Then, go through all #includes to see if they match the regexes, // discarding the identity mappings. TODO(wan): to improve // performance, don't construct more than one Regex object for each // element in the above vectors. for (const auto& incmap : quoted_includes_to_quoted_includers_) { const string& hdr = incmap.first; for (const string& regex_key : filepath_include_map_regex_keys) { const string regex = regex_key.substr(1); const vector& map_to = filepath_include_map_[regex_key]; if (RegexMatch(regex_dialect, hdr, regex) && !ContainsQuotedInclude(map_to, hdr)) { for (const MappedInclude& target : map_to) { filepath_include_map_[hdr].push_back(MappedInclude( RegexReplace(regex_dialect, hdr, regex, target.quoted_include))); } MarkVisibility(&include_visibility_map_, hdr, include_visibility_map_[regex_key]); } } for (const string& regex_key : friend_to_headers_map_regex_keys) { if (RegexMatch(regex_dialect, hdr, regex_key.substr(1))) { InsertAllInto(friend_to_headers_map_[regex_key], &friend_to_headers_map_[hdr]); } } } } // Handle work that's best done after we've seen all the mappings // (including dynamically-added ones) and all the include files. // For instance, we can now expand all the regexes we've seen in // the mapping-keys, since we have the full list of #includes to // match them again. We also transitively-close the maps. void IncludePicker::FinalizeAddedIncludes() { CHECK_(!has_called_finalize_added_include_lines_ && "Can't call FAI twice"); // The map keys may be regular expressions. // Match those to seen #includes now. ExpandRegexes(); // If a.h maps to b.h maps to c.h, we'd like an entry from a.h to c.h too. MakeMapTransitive(&filepath_include_map_); // Now that filepath_include_map_ is transitively closed, it's an // easy task to get the values of symbol_include_map_ closed too. for (IncludeMap::value_type& symbol_include : symbol_include_map_) { ExpandOnce(filepath_include_map_, &symbol_include.second); } has_called_finalize_added_include_lines_ = true; } // For the given key, return the vector of values associated with that // key, or an empty vector if the key does not exist in the map. // *However*, we filter out all values that have private visibility // before returning the vector. *Also*, if the key is public in // the map, we insert the key as the first of the returned values, // this is an implicit "self-map." vector IncludePicker::GetPublicValues( const IncludePicker::IncludeMap& m, const string& key) const { CHECK_(!StartsWith(key, "@")); vector retval; const vector* values = FindInMap(&m, key); if (!values || values->empty()) return retval; for (const MappedInclude& value : *values) { CHECK_(!StartsWith(value.quoted_include, "@")); if (GetVisibility(value, kPublic) == kPublic) retval.push_back(value); } return retval; } string IncludePicker::MaybeGetIncludeNameAsWritten( const string& includer_filepath, const string& includee_filepath) const { const pair key(includer_filepath, includee_filepath); // I want to use GetOrDefault here, but it has trouble deducing tpl args. const string* value = FindInMap(&includer_and_includee_to_include_as_written_, key); return value ? *value : ""; } vector IncludePicker::BestQuotedIncludesForIncluder( const vector& includes, const string& including_filepath) const { // Convert each MappedInclude to a quoted include, according to the // following priorities: // 1. If the file is already included, use whatever name it's already // included via. This is better to use than ConvertToQuotedInclude // because it avoids trouble when the same file is accessible via // different include search-paths, or is accessed via a symlink. // 2. If the quoted include in the MappedInclude object is an absolute path, // that's unlikely to be what's wanted. Try to convert it to a relative // include via ConvertToQuotedInclude. // 3. Otherwise, use the quoted include present in the MappedInclude. const string including_path = MakeAbsolutePath(GetParentPath(including_filepath)); vector retval; for (const MappedInclude& mapped_include : includes) { const string& quoted_include_as_written = MaybeGetIncludeNameAsWritten(including_filepath, mapped_include.path); if (!quoted_include_as_written.empty()) { retval.push_back(quoted_include_as_written); } else if (mapped_include.HasAbsoluteQuotedInclude() && !mapped_include.path.empty()) { retval.push_back(ConvertToQuotedInclude(mapped_include.path, including_path)); } else { retval.push_back(mapped_include.quoted_include); } } return retval; } vector IncludePicker::GetCandidateHeadersForSymbol( const string& symbol) const { CHECK_(has_called_finalize_added_include_lines_ && "Must finalize includes"); return GetPublicValues(symbol_include_map_, symbol); } vector IncludePicker::GetCandidateHeadersForSymbolUsedFrom( const string& symbol, const string& including_filepath) const { return BestQuotedIncludesForIncluder( GetCandidateHeadersForSymbol(symbol), including_filepath); } vector IncludePicker::GetCandidateHeadersForFilepath( const string& filepath, const string& including_filepath) const { CHECK_(has_called_finalize_added_include_lines_ && "Must finalize includes"); string absolute_quoted_header = ConvertToQuotedInclude(filepath); vector retval = GetPublicValues(filepath_include_map_, absolute_quoted_header); // We also need to consider the header itself. Make that an option if it's // public or there's no other option. string quoted_header; if (including_filepath.empty()) { quoted_header = absolute_quoted_header; } else { quoted_header = ConvertToQuotedInclude( filepath, MakeAbsolutePath(GetParentPath(including_filepath))); } MappedInclude default_header(quoted_header, filepath); if (retval.empty() || GetVisibility(default_header, kPublic) == kPublic) { // Insert at front so it's the preferred option retval.insert(retval.begin(), default_header); } return retval; } // Except for the case that the includer is a 'friend' of the includee // (via an '// IWYU pragma: friend XXX'), the same as // GetCandidateHeadersForFilepath. vector IncludePicker::GetCandidateHeadersForFilepathIncludedFrom( const string& included_filepath, const string& including_filepath) const { vector mapped_includes; // We pass the own files path to ConvertToQuotedInclude so the quoted include // for the case that there is no matching `-I` option is just the filename // (e.g. "foo.cpp") instead of the absolute file path. const string including_path = MakeAbsolutePath(GetParentPath(including_filepath)); const string quoted_includer = ConvertToQuotedInclude( including_filepath, including_path); const string quoted_includee = ConvertToQuotedInclude( included_filepath, including_path); const set* headers_with_includer_as_friend = FindInMap(&friend_to_headers_map_, quoted_includer); if (headers_with_includer_as_friend != nullptr && ContainsKey(*headers_with_includer_as_friend, included_filepath)) { mapped_includes.push_back( MappedInclude(quoted_includee, including_filepath)); } else { mapped_includes = GetCandidateHeadersForFilepath(included_filepath, including_filepath); if (mapped_includes.size() == 1) { if (GetVisibility(mapped_includes[0]) == kPrivate) { VERRS(0) << "Warning: " << "No public header found to replace the private header " << included_filepath << "\n"; } } } return BestQuotedIncludesForIncluder(mapped_includes, including_filepath); } bool IncludePicker::HasMapping(const string& map_from_filepath, const string& map_to_filepath) const { CHECK_(has_called_finalize_added_include_lines_ && "Must finalize includes"); const string quoted_from = ConvertToQuotedInclude(map_from_filepath); const string quoted_to = ConvertToQuotedInclude(map_to_filepath); // We can't use GetCandidateHeadersForFilepath since includer might be private const vector* all_mappers = FindInMap(&filepath_include_map_, quoted_from); if (all_mappers) { if (ContainsQuotedInclude(*all_mappers, quoted_to)) { return true; } } return quoted_to == quoted_from; // indentity mapping, why not? } bool IncludePicker::IsPublic(const clang::FileEntry* file) const { CHECK_(file && "Need existing FileEntry"); const string path = GetFilePath(file); const string quoted_file = ConvertToQuotedInclude(path); const MappedInclude include(quoted_file, path); return (GetVisibility(include) == kPublic); } // Parses a YAML/JSON file containing mapping directives of various types. void IncludePicker::AddMappingsFromFile(const string& filename) { vector default_search_path; return AddMappingsFromFile(filename, default_search_path); } // Parses a YAML/JSON file containing mapping directives of various types: // symbol - symbol name -> quoted include // include - private quoted include -> public quoted include // ref - include mechanism for mapping files, to allow project-specific // groupings // This private implementation method is recursive and builds the search path // incrementally. void IncludePicker::AddMappingsFromFile(const string& filename, const vector& search_path) { string absolute_path = FindFileInSearchPath(search_path, filename); llvm::ErrorOr> bufferOrError = MemoryBuffer::getFile(absolute_path); if (std::error_code error = bufferOrError.getError()) { VERRS(0) << "Cannot open mapping file '" << absolute_path << "': " << error.message() << ".\n"; return; } VERRS(5) << "Adding mappings from file '" << absolute_path << "'.\n"; SourceMgr source_manager; Stream json_stream(bufferOrError.get()->getMemBufferRef(), source_manager); document_iterator stream_begin = json_stream.begin(); if (stream_begin == json_stream.end()) return; // Get root sequence. Node* root = stream_begin->getRoot(); SequenceNode *array = llvm::dyn_cast(root); if (array == nullptr) { json_stream.printError(root, "Root element must be an array."); return; } for (Node& array_item_node : *array) { Node* current_node = &array_item_node; // Every item must be a JSON object ("mapping" in YAML terms.) MappingNode* mapping = llvm::dyn_cast(current_node); if (mapping == nullptr) { json_stream.printError(current_node, "Mapping directives must be objects."); return; } for (KeyValueNode &mapping_item_node : *mapping) { // General form is { directive: }. const string directive = GetScalarValue(mapping_item_node.getKey()); if (directive == "symbol") { // Symbol mapping. vector mapping = GetSequenceValue(mapping_item_node.getValue()); if (mapping.size() != 4) { json_stream.printError(current_node, "Symbol mapping expects a value on the form " "'[from, visibility, to, visibility]'."); return; } // Ignore unused from-visibility, at some point maybe remove it from the // mapping file format. IncludeVisibility to_visibility = ParseVisibility(mapping[3]); if (to_visibility == kUnusedVisibility) { json_stream.printError(current_node, "Unknown visibility '" + mapping[3] + "'."); return; } if (!IsQuotedInclude(mapping[2])) { json_stream.printError( current_node, "Expected to-entry to be quoted include, but was '" + mapping[2] + "'"); return; } AddSymbolMapping(mapping[0], MappedInclude(mapping[2]), to_visibility); } else if (directive == "include") { // Include mapping. vector mapping = GetSequenceValue(mapping_item_node.getValue()); if (mapping.size() != 4) { json_stream.printError(current_node, "Include mapping expects a value on the form " "'[from, visibility, to, visibility]'."); return; } IncludeVisibility from_visibility = ParseVisibility(mapping[1]); if (from_visibility == kUnusedVisibility) { json_stream.printError(current_node, "Unknown visibility '" + mapping[1] + "'."); return; } IncludeVisibility to_visibility = ParseVisibility(mapping[3]); if (to_visibility == kUnusedVisibility) { json_stream.printError(current_node, "Unknown visibility '" + mapping[3] + "'."); return; } if (!IsQuotedFilepathPattern(mapping[0])) { json_stream.printError( current_node, "Expected from-entry to be quoted filepath or @regex, but was '" + mapping[0] + "'"); return; } if (!IsQuotedInclude(mapping[2])) { json_stream.printError( current_node, "Expected to-entry to be quoted include, but was '" + mapping[2] + "'"); return; } AddIncludeMapping(mapping[0], from_visibility, MappedInclude(mapping[2]), to_visibility); } else if (directive == "ref") { // Mapping ref. string ref_file = GetScalarValue(mapping_item_node.getValue()); if (ref_file.empty()) { json_stream.printError(current_node, "Mapping ref expects a single filename value."); return; } // Add the path of the file we're currently processing // to the search path. Allows refs to be relative to referrer. vector extended_search_path = ExtendMappingFileSearchPath(search_path, GetParentPath(absolute_path)); // Recurse. AddMappingsFromFile(ref_file, extended_search_path); } else { json_stream.printError(current_node, "Unknown directive '" + directive + "'."); return; } } } } IncludeVisibility IncludePicker::ParseVisibility( const string& visibility) const { if (visibility == "private") return kPrivate; else if (visibility == "public") return kPublic; return kUnusedVisibility; } IncludeVisibility IncludePicker::GetVisibility( const MappedInclude& include, IncludeVisibility default_value) const { const IncludeVisibility* include_visibility = FindInMap(&include_visibility_map_, include.quoted_include); if (include_visibility) { return *include_visibility; } return GetOrDefault(path_visibility_map_, include.path, default_value); } } // namespace include_what_you_use