Loading...
1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3#
4# Copyright (C) Google LLC, 2018
5#
6# Author: Tom Roeder <tmroeder@google.com>
7#
8"""A tool for generating compile_commands.json in the Linux kernel."""
9
10import argparse
11import json
12import logging
13import os
14import re
15import subprocess
16import sys
17
18_DEFAULT_OUTPUT = 'compile_commands.json'
19_DEFAULT_LOG_LEVEL = 'WARNING'
20
21_FILENAME_PATTERN = r'^\..*\.cmd$'
22_LINE_PATTERN = r'^(saved)?cmd_[^ ]*\.o := (?P<command_prefix>.* )(?P<file_path>[^ ]*\.[cS]) *(;|$)'
23_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
24# The tools/ directory adopts a different build system, and produces .cmd
25# files in a different format. Do not support it.
26_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
27
28def parse_arguments():
29 """Sets up and parses command-line arguments.
30
31 Returns:
32 log_level: A logging level to filter log output.
33 directory: The work directory where the objects were built.
34 ar: Command used for parsing .a archives.
35 output: Where to write the compile-commands JSON file.
36 paths: The list of files/directories to handle to find .cmd files.
37 """
38 usage = 'Creates a compile_commands.json database from kernel .cmd files'
39 parser = argparse.ArgumentParser(description=usage)
40
41 directory_help = ('specify the output directory used for the kernel build '
42 '(defaults to the working directory)')
43 parser.add_argument('-d', '--directory', type=str, default='.',
44 help=directory_help)
45
46 output_help = ('path to the output command database (defaults to ' +
47 _DEFAULT_OUTPUT + ')')
48 parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
49 help=output_help)
50
51 log_level_help = ('the level of log messages to produce (defaults to ' +
52 _DEFAULT_LOG_LEVEL + ')')
53 parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
54 default=_DEFAULT_LOG_LEVEL, help=log_level_help)
55
56 ar_help = 'command used for parsing .a archives'
57 parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help)
58
59 paths_help = ('directories to search or files to parse '
60 '(files should be *.o, *.a, or modules.order). '
61 'If nothing is specified, the current directory is searched')
62 parser.add_argument('paths', type=str, nargs='*', help=paths_help)
63
64 args = parser.parse_args()
65
66 return (args.log_level,
67 os.path.realpath(args.directory),
68 args.output,
69 args.ar,
70 args.paths if len(args.paths) > 0 else [args.directory])
71
72
73def cmdfiles_in_dir(directory):
74 """Generate the iterator of .cmd files found under the directory.
75
76 Walk under the given directory, and yield every .cmd file found.
77
78 Args:
79 directory: The directory to search for .cmd files.
80
81 Yields:
82 The path to a .cmd file.
83 """
84
85 filename_matcher = re.compile(_FILENAME_PATTERN)
86 exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
87
88 for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
89 # Prune unwanted directories.
90 if dirpath in exclude_dirs:
91 dirnames[:] = []
92 continue
93
94 for filename in filenames:
95 if filename_matcher.match(filename):
96 yield os.path.join(dirpath, filename)
97
98
99def to_cmdfile(path):
100 """Return the path of .cmd file used for the given build artifact
101
102 Args:
103 Path: file path
104
105 Returns:
106 The path to .cmd file
107 """
108 dir, base = os.path.split(path)
109 return os.path.join(dir, '.' + base + '.cmd')
110
111
112def cmdfiles_for_a(archive, ar):
113 """Generate the iterator of .cmd files associated with the archive.
114
115 Parse the given archive, and yield every .cmd file used to build it.
116
117 Args:
118 archive: The archive to parse
119
120 Yields:
121 The path to every .cmd file found
122 """
123 for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
124 yield to_cmdfile(obj)
125
126
127def cmdfiles_for_modorder(modorder):
128 """Generate the iterator of .cmd files associated with the modules.order.
129
130 Parse the given modules.order, and yield every .cmd file used to build the
131 contained modules.
132
133 Args:
134 modorder: The modules.order file to parse
135
136 Yields:
137 The path to every .cmd file found
138 """
139 with open(modorder) as f:
140 for line in f:
141 obj = line.rstrip()
142 base, ext = os.path.splitext(obj)
143 if ext != '.o':
144 sys.exit('{}: module path must end with .o'.format(obj))
145 mod = base + '.mod'
146 # Read from *.mod, to get a list of objects that compose the module.
147 with open(mod) as m:
148 for mod_line in m:
149 yield to_cmdfile(mod_line.rstrip())
150
151
152def process_line(root_directory, command_prefix, file_path):
153 """Extracts information from a .cmd line and creates an entry from it.
154
155 Args:
156 root_directory: The directory that was searched for .cmd files. Usually
157 used directly in the "directory" entry in compile_commands.json.
158 command_prefix: The extracted command line, up to the last element.
159 file_path: The .c file from the end of the extracted command.
160 Usually relative to root_directory, but sometimes absolute.
161
162 Returns:
163 An entry to append to compile_commands.
164
165 Raises:
166 ValueError: Could not find the extracted file based on file_path and
167 root_directory or file_directory.
168 """
169 # The .cmd files are intended to be included directly by Make, so they
170 # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
171 # kernel version). The compile_commands.json file is not interepreted
172 # by Make, so this code replaces the escaped version with '#'.
173 prefix = command_prefix.replace(r'\#', '#').replace('$(pound)', '#')
174
175 # Return the canonical path, eliminating any symbolic links encountered in the path.
176 abs_path = os.path.realpath(os.path.join(root_directory, file_path))
177 if not os.path.exists(abs_path):
178 raise ValueError('File %s not found' % abs_path)
179 return {
180 'directory': root_directory,
181 'file': abs_path,
182 'command': prefix + file_path,
183 }
184
185
186def main():
187 """Walks through the directory and finds and parses .cmd files."""
188 log_level, directory, output, ar, paths = parse_arguments()
189
190 level = getattr(logging, log_level)
191 logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
192
193 line_matcher = re.compile(_LINE_PATTERN)
194
195 compile_commands = []
196
197 for path in paths:
198 # If 'path' is a directory, handle all .cmd files under it.
199 # Otherwise, handle .cmd files associated with the file.
200 # built-in objects are linked via vmlinux.a
201 # Modules are listed in modules.order.
202 if os.path.isdir(path):
203 cmdfiles = cmdfiles_in_dir(path)
204 elif path.endswith('.a'):
205 cmdfiles = cmdfiles_for_a(path, ar)
206 elif path.endswith('modules.order'):
207 cmdfiles = cmdfiles_for_modorder(path)
208 else:
209 sys.exit('{}: unknown file type'.format(path))
210
211 for cmdfile in cmdfiles:
212 with open(cmdfile, 'rt') as f:
213 result = line_matcher.match(f.readline())
214 if result:
215 try:
216 entry = process_line(directory, result.group('command_prefix'),
217 result.group('file_path'))
218 compile_commands.append(entry)
219 except ValueError as err:
220 logging.info('Could not add line from %s: %s',
221 cmdfile, err)
222
223 with open(output, 'wt') as f:
224 json.dump(sorted(compile_commands, key=lambda x: x["file"]), f, indent=2, sort_keys=True)
225
226
227if __name__ == '__main__':
228 main()
1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3#
4# Copyright (C) Google LLC, 2018
5#
6# Author: Tom Roeder <tmroeder@google.com>
7#
8"""A tool for generating compile_commands.json in the Linux kernel."""
9
10import argparse
11import json
12import logging
13import os
14import re
15import subprocess
16import sys
17
18_DEFAULT_OUTPUT = 'compile_commands.json'
19_DEFAULT_LOG_LEVEL = 'WARNING'
20
21_FILENAME_PATTERN = r'^\..*\.cmd$'
22_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
23_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
24# The tools/ directory adopts a different build system, and produces .cmd
25# files in a different format. Do not support it.
26_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
27
28def parse_arguments():
29 """Sets up and parses command-line arguments.
30
31 Returns:
32 log_level: A logging level to filter log output.
33 directory: The work directory where the objects were built.
34 ar: Command used for parsing .a archives.
35 output: Where to write the compile-commands JSON file.
36 paths: The list of files/directories to handle to find .cmd files.
37 """
38 usage = 'Creates a compile_commands.json database from kernel .cmd files'
39 parser = argparse.ArgumentParser(description=usage)
40
41 directory_help = ('specify the output directory used for the kernel build '
42 '(defaults to the working directory)')
43 parser.add_argument('-d', '--directory', type=str, default='.',
44 help=directory_help)
45
46 output_help = ('path to the output command database (defaults to ' +
47 _DEFAULT_OUTPUT + ')')
48 parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
49 help=output_help)
50
51 log_level_help = ('the level of log messages to produce (defaults to ' +
52 _DEFAULT_LOG_LEVEL + ')')
53 parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
54 default=_DEFAULT_LOG_LEVEL, help=log_level_help)
55
56 ar_help = 'command used for parsing .a archives'
57 parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help)
58
59 paths_help = ('directories to search or files to parse '
60 '(files should be *.o, *.a, or modules.order). '
61 'If nothing is specified, the current directory is searched')
62 parser.add_argument('paths', type=str, nargs='*', help=paths_help)
63
64 args = parser.parse_args()
65
66 return (args.log_level,
67 os.path.abspath(args.directory),
68 args.output,
69 args.ar,
70 args.paths if len(args.paths) > 0 else [args.directory])
71
72
73def cmdfiles_in_dir(directory):
74 """Generate the iterator of .cmd files found under the directory.
75
76 Walk under the given directory, and yield every .cmd file found.
77
78 Args:
79 directory: The directory to search for .cmd files.
80
81 Yields:
82 The path to a .cmd file.
83 """
84
85 filename_matcher = re.compile(_FILENAME_PATTERN)
86 exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
87
88 for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
89 # Prune unwanted directories.
90 if dirpath in exclude_dirs:
91 dirnames[:] = []
92 continue
93
94 for filename in filenames:
95 if filename_matcher.match(filename):
96 yield os.path.join(dirpath, filename)
97
98
99def to_cmdfile(path):
100 """Return the path of .cmd file used for the given build artifact
101
102 Args:
103 Path: file path
104
105 Returns:
106 The path to .cmd file
107 """
108 dir, base = os.path.split(path)
109 return os.path.join(dir, '.' + base + '.cmd')
110
111
112def cmdfiles_for_o(obj):
113 """Generate the iterator of .cmd files associated with the object
114
115 Yield the .cmd file used to build the given object
116
117 Args:
118 obj: The object path
119
120 Yields:
121 The path to .cmd file
122 """
123 yield to_cmdfile(obj)
124
125
126def cmdfiles_for_a(archive, ar):
127 """Generate the iterator of .cmd files associated with the archive.
128
129 Parse the given archive, and yield every .cmd file used to build it.
130
131 Args:
132 archive: The archive to parse
133
134 Yields:
135 The path to every .cmd file found
136 """
137 for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
138 yield to_cmdfile(obj)
139
140
141def cmdfiles_for_modorder(modorder):
142 """Generate the iterator of .cmd files associated with the modules.order.
143
144 Parse the given modules.order, and yield every .cmd file used to build the
145 contained modules.
146
147 Args:
148 modorder: The modules.order file to parse
149
150 Yields:
151 The path to every .cmd file found
152 """
153 with open(modorder) as f:
154 for line in f:
155 ko = line.rstrip()
156 base, ext = os.path.splitext(ko)
157 if ext != '.ko':
158 sys.exit('{}: module path must end with .ko'.format(ko))
159 mod = base + '.mod'
160 # The first line of *.mod lists the objects that compose the module.
161 with open(mod) as m:
162 for obj in m.readline().split():
163 yield to_cmdfile(obj)
164
165
166def process_line(root_directory, command_prefix, file_path):
167 """Extracts information from a .cmd line and creates an entry from it.
168
169 Args:
170 root_directory: The directory that was searched for .cmd files. Usually
171 used directly in the "directory" entry in compile_commands.json.
172 command_prefix: The extracted command line, up to the last element.
173 file_path: The .c file from the end of the extracted command.
174 Usually relative to root_directory, but sometimes absolute.
175
176 Returns:
177 An entry to append to compile_commands.
178
179 Raises:
180 ValueError: Could not find the extracted file based on file_path and
181 root_directory or file_directory.
182 """
183 # The .cmd files are intended to be included directly by Make, so they
184 # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
185 # kernel version). The compile_commands.json file is not interepreted
186 # by Make, so this code replaces the escaped version with '#'.
187 prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
188
189 # Use os.path.abspath() to normalize the path resolving '.' and '..' .
190 abs_path = os.path.abspath(os.path.join(root_directory, file_path))
191 if not os.path.exists(abs_path):
192 raise ValueError('File %s not found' % abs_path)
193 return {
194 'directory': root_directory,
195 'file': abs_path,
196 'command': prefix + file_path,
197 }
198
199
200def main():
201 """Walks through the directory and finds and parses .cmd files."""
202 log_level, directory, output, ar, paths = parse_arguments()
203
204 level = getattr(logging, log_level)
205 logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
206
207 line_matcher = re.compile(_LINE_PATTERN)
208
209 compile_commands = []
210
211 for path in paths:
212 # If 'path' is a directory, handle all .cmd files under it.
213 # Otherwise, handle .cmd files associated with the file.
214 # Most of built-in objects are linked via archives (built-in.a or lib.a)
215 # but some objects are linked to vmlinux directly.
216 # Modules are listed in modules.order.
217 if os.path.isdir(path):
218 cmdfiles = cmdfiles_in_dir(path)
219 elif path.endswith('.o'):
220 cmdfiles = cmdfiles_for_o(path)
221 elif path.endswith('.a'):
222 cmdfiles = cmdfiles_for_a(path, ar)
223 elif path.endswith('modules.order'):
224 cmdfiles = cmdfiles_for_modorder(path)
225 else:
226 sys.exit('{}: unknown file type'.format(path))
227
228 for cmdfile in cmdfiles:
229 with open(cmdfile, 'rt') as f:
230 result = line_matcher.match(f.readline())
231 if result:
232 try:
233 entry = process_line(directory, result.group(1),
234 result.group(2))
235 compile_commands.append(entry)
236 except ValueError as err:
237 logging.info('Could not add line from %s: %s',
238 cmdfile, err)
239
240 with open(output, 'wt') as f:
241 json.dump(compile_commands, f, indent=2, sort_keys=True)
242
243
244if __name__ == '__main__':
245 main()