1#!/usr/bin/env python
  2# SPDX-License-Identifier: GPL-2.0
  3#
  4# Copyright (C) Google LLC, 2018
  5#
  6# Author: Tom Roeder <tmroeder@google.com>
  7#
  8"""A tool for generating compile_commands.json in the Linux kernel."""
  9
 10import argparse
 11import json
 12import logging
 13import os
 14import re
 15
 16_DEFAULT_OUTPUT = 'compile_commands.json'
 17_DEFAULT_LOG_LEVEL = 'WARNING'
 18
 19_FILENAME_PATTERN = r'^\..*\.cmd$'
 20_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
 21_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
 22
 23# A kernel build generally has over 2000 entries in its compile_commands.json
 24# database. If this code finds 300 or fewer, then warn the user that they might
 25# not have all the .cmd files, and they might need to compile the kernel.
 26_LOW_COUNT_THRESHOLD = 300
 27
 28
 29def parse_arguments():
 30    """Sets up and parses command-line arguments.
 31
 32    Returns:
 33        log_level: A logging level to filter log output.
 34        directory: The directory to search for .cmd files.
 35        output: Where to write the compile-commands JSON file.
 36    """
 37    usage = 'Creates a compile_commands.json database from kernel .cmd files'
 38    parser = argparse.ArgumentParser(description=usage)
 39
 40    directory_help = ('Path to the kernel source directory to search '
 41                      '(defaults to the working directory)')
 42    parser.add_argument('-d', '--directory', type=str, help=directory_help)
 43
 44    output_help = ('The location to write compile_commands.json (defaults to '
 45                   'compile_commands.json in the search directory)')
 46    parser.add_argument('-o', '--output', type=str, help=output_help)
 47
 48    log_level_help = ('The level of log messages to produce (one of ' +
 49                      ', '.join(_VALID_LOG_LEVELS) + '; defaults to ' +
 50                      _DEFAULT_LOG_LEVEL + ')')
 51    parser.add_argument(
 52        '--log_level', type=str, default=_DEFAULT_LOG_LEVEL,
 53        help=log_level_help)
 54
 55    args = parser.parse_args()
 56
 57    log_level = args.log_level
 58    if log_level not in _VALID_LOG_LEVELS:
 59        raise ValueError('%s is not a valid log level' % log_level)
 60
 61    directory = args.directory or os.getcwd()
 62    output = args.output or os.path.join(directory, _DEFAULT_OUTPUT)
 63    directory = os.path.abspath(directory)
 64
 65    return log_level, directory, output
 66
 67
 68def process_line(root_directory, file_directory, command_prefix, relative_path):
 69    """Extracts information from a .cmd line and creates an entry from it.
 70
 71    Args:
 72        root_directory: The directory that was searched for .cmd files. Usually
 73            used directly in the "directory" entry in compile_commands.json.
 74        file_directory: The path to the directory the .cmd file was found in.
 75        command_prefix: The extracted command line, up to the last element.
 76        relative_path: The .c file from the end of the extracted command.
 77            Usually relative to root_directory, but sometimes relative to
 78            file_directory and sometimes neither.
 79
 80    Returns:
 81        An entry to append to compile_commands.
 82
 83    Raises:
 84        ValueError: Could not find the extracted file based on relative_path and
 85            root_directory or file_directory.
 86    """
 87    # The .cmd files are intended to be included directly by Make, so they
 88    # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
 89    # kernel version). The compile_commands.json file is not interepreted
 90    # by Make, so this code replaces the escaped version with '#'.
 91    prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
 92
 93    cur_dir = root_directory
 94    expected_path = os.path.join(cur_dir, relative_path)
 95    if not os.path.exists(expected_path):
 96        # Try using file_directory instead. Some of the tools have a different
 97        # style of .cmd file than the kernel.
 98        cur_dir = file_directory
 99        expected_path = os.path.join(cur_dir, relative_path)
100        if not os.path.exists(expected_path):
101            raise ValueError('File %s not in %s or %s' %
102                             (relative_path, root_directory, file_directory))
103    return {
104        'directory': cur_dir,
105        'file': relative_path,
106        'command': prefix + relative_path,
107    }
108
109
110def main():
111    """Walks through the directory and finds and parses .cmd files."""
112    log_level, directory, output = parse_arguments()
113
114    level = getattr(logging, log_level)
115    logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
116
117    filename_matcher = re.compile(_FILENAME_PATTERN)
118    line_matcher = re.compile(_LINE_PATTERN)
119
120    compile_commands = []
121    for dirpath, _, filenames in os.walk(directory):
122        for filename in filenames:
123            if not filename_matcher.match(filename):
124                continue
125            filepath = os.path.join(dirpath, filename)
126
127            with open(filepath, 'rt') as f:
128                for line in f:
129                    result = line_matcher.match(line)
130                    if not result:
131                        continue
132
133                    try:
134                        entry = process_line(directory, dirpath,
135                                             result.group(1), result.group(2))
136                        compile_commands.append(entry)
137                    except ValueError as err:
138                        logging.info('Could not add line from %s: %s',
139                                     filepath, err)
140
141    with open(output, 'wt') as f:
142        json.dump(compile_commands, f, indent=2, sort_keys=True)
143
144    count = len(compile_commands)
145    if count < _LOW_COUNT_THRESHOLD:
146        logging.warning(
147            'Found %s entries. Have you compiled the kernel?', count)
148
149
150if __name__ == '__main__':
151    main()