1 /*
  2  *  Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
  3  *  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  *  This code is free software; you can redistribute it and/or modify it
  6  *  under the terms of the GNU General Public License version 2 only, as
  7  *  published by the Free Software Foundation.  Oracle designates this
  8  *  particular file as subject to the "Classpath" exception as provided
  9  *  by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  *  This code is distributed in the hope that it will be useful, but WITHOUT
 12  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  *  version 2 for more details (a copy is included in the LICENSE file that
 15  *  accompanied this code).
 16  *
 17  *  You should have received a copy of the GNU General Public License version
 18  *  2 along with this work; if not, write to the Free Software Foundation,
 19  *  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  *   Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  *  or visit www.oracle.com if you need additional information or have any
 23  *  questions.
 24  *
 25  */
 26 
 27 package jdk.internal.jextract.impl;
 28 
 29 import jdk.incubator.jextract.Declaration;
 30 import jdk.incubator.jextract.Position;
 31 import jdk.incubator.jextract.Type;
 32 import jdk.incubator.jextract.JextractTool;
 33 import jdk.internal.clang.Cursor;
 34 import jdk.internal.clang.CursorKind;
 35 import jdk.internal.clang.Diagnostic;
 36 import jdk.internal.clang.EvalResult;
 37 import jdk.internal.clang.Index;
 38 import jdk.internal.clang.LibClang;
 39 import jdk.internal.clang.TranslationUnit;
 40 import jdk.internal.clang.TypeKind;
 41 
 42 import java.io.IOException;
 43 import java.nio.file.Files;
 44 import java.nio.file.Path;
 45 import java.util.Collection;
 46 import java.util.LinkedHashMap;
 47 import java.util.List;
 48 import java.util.Map;
 49 import java.util.Optional;
 50 import java.util.stream.Collectors;
 51 import java.util.stream.Stream;
 52 
 53 class MacroParserImpl {
 54 
 55     private final ClangReparser reparser;
 56     private final TreeMaker treeMaker;
 57     final MacroTable macroTable;
 58 
 59     private MacroParserImpl(ClangReparser reparser, TreeMaker treeMaker) {
 60         this.reparser = reparser;
 61         this.treeMaker = treeMaker;
 62         this.macroTable = new MacroTable();
 63     }
 64 
 65     static MacroParserImpl make(TreeMaker treeMaker, TranslationUnit tu, Collection<String> args) {
 66         ClangReparser reparser;
 67         try {
 68             reparser = new ClangReparser(tu, args);
 69         } catch (IOException | Index.ParsingFailedException ex) {
 70             throw new RuntimeException(ex);
 71         }
 72 
 73         return new MacroParserImpl(reparser, treeMaker);
 74     }
 75 
 76     /**
 77      * This method attempts to evaluate the macro. Evaluation occurs in two steps: first, an attempt is made
 78      * to see if the macro corresponds to a simple numeric constant. If so, the constant is parsed in Java directly.
 79      * If that is not possible (e.g. because the macro refers to other macro, or has a more complex grammar), fall
 80      * back to use clang evaluation support.
 81      */
 82     Optional<Declaration.Constant> parseConstant(Position pos, String name, String[] tokens) {
 83         if (!(pos instanceof TreeMaker.CursorPosition)) {
 84             return Optional.empty();
 85         } else {
 86             Cursor cursor = ((TreeMaker.CursorPosition)pos).cursor();
 87             if (cursor.isMacroFunctionLike()) {
 88                 return Optional.empty();
 89             } else if (tokens.length == 2) {
 90                 //check for fast path
 91                 Integer num = toNumber(tokens[1]);
 92                 if (num != null) {
 93                     return Optional.of(treeMaker.createMacro(cursor, name, Type.primitive(Type.Primitive.Kind.Int), (long)num));
 94                 }
 95             }
 96             macroTable.enterMacro(name, tokens, cursor);
 97             return Optional.empty();
 98         }
 99     }
100 
101     private Integer toNumber(String str) {
102         try {
103             // Integer.decode supports '#' hex literals which is not valid in C.
104             return str.length() > 0 && str.charAt(0) != '#'? Integer.decode(str) : null;
105         } catch (NumberFormatException nfe) {
106             return null;
107         }
108     }
109 
110     /**
111      * This class allows client to reparse a snippet of code against a given set of include files.
112      * For performance reasons, the set of includes (which comes from the jextract parser) is compiled
113      * into a precompiled header, so as to speed to incremental recompilation of the generated snippets.
114      */
115     static class ClangReparser {
116         final Path macro;
117         final Index macroIndex = LibClang.createIndex(true);
118         final TranslationUnit macroUnit;
119 
120         public ClangReparser(TranslationUnit tu, Collection<String> args) throws IOException, Index.ParsingFailedException {
121             Path precompiled = Files.createTempFile("jextract$", ".pch");
122             precompiled.toFile().deleteOnExit();
123             tu.save(precompiled);
124             this.macro = Files.createTempFile("jextract$", ".h");
125             this.macro.toFile().deleteOnExit();
126             String[] patchedArgs = Stream.concat(
127                 Stream.of(
128                     // Avoid system search path, use bundled instead
129                     "-nostdinc",
130                     "-ferror-limit=0",
131                     // precompiled header
132                     "-include-pch", precompiled.toAbsolutePath().toString()),
133                 args.stream()).toArray(String[]::new);
134             this.macroUnit = macroIndex.parse(macro.toAbsolutePath().toString(),
135                     this::processDiagnostics,
136                     false, //add serialization support (needed for macros)
137                     patchedArgs);
138         }
139 
140         void processDiagnostics(Diagnostic diag) {
141             if (JextractTool.DEBUG) {
142                 System.err.println("Error while processing macro: " + diag.spelling());
143             }
144         }
145 
146         public Stream<Cursor> reparse(String snippet) {
147             macroUnit.reparse(this::processDiagnostics,
148                     Index.UnsavedFile.of(macro, snippet));
149             return macroUnit.getCursor().children();
150         }
151     }
152 
153     /**
154      * This abstraction is used to collect all macros which could not be interpreted during {@link #parseConstant(Position, String, String[])}.
155      * All unparsed macros in the table can have three different states: UNPARSED (which means the macro has not been parsed yet),
156      * SUCCESS (which means the macro has been parsed and has a type and a value) and FAILURE, which means the macro has been
157      * parsed with some errors, but for which we were at least able to infer a type.
158      *
159      * The reparsing process goes as follows:
160      * 1. all unparsed macros are added to the table in the UNPARSED state.
161      * 2. a snippet for all macros in the UNPARSED state is compiled and the table state is updated
162      * 3. a recovery snippet for all macros in the FAILURE state is compiled and the table state is updated again
163      * 4. we repeat from (2) until no further progress is made.
164      * 5. we return a list of macro which are in the SUCCESS state.
165      *
166      * State transitions in the table are as follows:
167      * - an UNPARSED macro can go to either SUCCESS, to FAILURE or be removed (if not even a type can be inferred)
168      * - a FAILURE macro can go to either SUCCESS (if recovery step succeds) or be removed
169      * - a SUCCESS macro cannot go in any other state
170      */
171     class MacroTable {
172 
173         final Map<String, Entry> macrosByMangledName = new LinkedHashMap<>();
174 
175         abstract class Entry {
176             final String name;
177             final String[] tokens;
178             final Cursor cursor;
179 
180             Entry(String name, String[] tokens, Cursor cursor) {
181                 this.name = name;
182                 this.tokens = tokens;
183                 this.cursor = cursor;
184             }
185 
186             String mangledName() {
187                 return "jextract$macro$" + name;
188             }
189 
190             Entry success(Type type, Object value) {
191                 throw new IllegalStateException();
192             }
193 
194             Entry failure(Type type) {
195                 throw new IllegalStateException();
196             }
197 
198             boolean isSuccess() {
199                 return false;
200             }
201             boolean isRecoverableFailure() {
202                 return false;
203             }
204             boolean isUnparsed() {
205                 return false;
206             }
207 
208             void update() {
209                 macrosByMangledName.put(mangledName(), this);
210             }
211         }
212 
213         class Unparsed extends Entry {
214             Unparsed(String name, String[] tokens, Cursor cursor) {
215                 super(name, tokens, cursor);
216             }
217 
218             @Override
219             Entry success(Type type, Object value) {
220                 return new Success(name, tokens, cursor, type, value);
221             }
222 
223             @Override
224             Entry failure(Type type) {
225                 return type != null ?
226                         new RecoverableFailure(name, tokens, cursor, type) :
227                         new UnparseableMacro(name, tokens, cursor);
228             }
229 
230             @Override
231             boolean isUnparsed() {
232                 return true;
233             }
234 
235             @Override
236             void update() {
237                 throw new IllegalStateException();
238             }
239         }
240 
241         class RecoverableFailure extends Entry {
242 
243             final Type type;
244 
245             public RecoverableFailure(String name, String[] tokens, Cursor cursor, Type type) {
246                 super(name, tokens, cursor);
247                 this.type = type;
248             }
249 
250             @Override
251             Entry success(Type type, Object value) {
252                 return new Success(name, tokens, cursor, this.type, value);
253             }
254 
255             @Override
256             Entry failure(Type type) {
257                 return new UnparseableMacro(name, tokens, cursor);
258             }
259 
260             @Override
261             boolean isRecoverableFailure() {
262                 return true;
263             }
264         }
265 
266         class Success extends Entry {
267             final Type type;
268             final Object value;
269 
270             public Success(String name, String[] tokens, Cursor cursor, Type type, Object value) {
271                 super(name, tokens, cursor);
272                 this.type = type;
273                 this.value = value;
274             }
275 
276             @Override
277             boolean isSuccess() {
278                 return true;
279             }
280 
281             public Object value() {
282                 return value;
283             }
284         }
285 
286         class UnparseableMacro extends Entry {
287 
288             UnparseableMacro(String name, String[] tokens, Cursor cursor) {
289                 super(name, tokens, cursor);
290             }
291 
292             @Override
293             void update() {
294                 macrosByMangledName.remove(mangledName());
295             }
296         };
297 
298         void enterMacro(String name, String[] tokens, Cursor cursor) {
299             Unparsed unparsed = new Unparsed(name, tokens, cursor);
300             macrosByMangledName.put(unparsed.mangledName(), unparsed);
301         }
302 
303         public List<Declaration.Constant> reparseConstants() {
304             int last = -1;
305             while (macrosByMangledName.size() > 0 && last != macrosByMangledName.size()) {
306                 last = macrosByMangledName.size();
307                 // step 1 - try parsing macros as var declarations
308                 reparseMacros(false);
309                 // step 2 - retry failed parsed macros as pointers
310                 reparseMacros(true);
311             }
312             treeMaker.typeMaker.resolveTypeReferences();
313             return macrosByMangledName.values().stream()
314                     .filter(Entry::isSuccess)
315                     .map(e -> treeMaker.createMacro(e.cursor, e.name, ((Success)e).type, ((Success)e).value))
316                     .collect(Collectors.toList());
317         }
318 
319         void updateTable(TypeMaker typeMaker, Cursor decl) {
320             String mangledName = decl.spelling();
321             Entry entry = macrosByMangledName.get(mangledName);
322             try (EvalResult result = decl.eval()) {
323                 Entry newEntry = switch (result.getKind()) {
324                     case Integral -> {
325                         long value = result.getAsInt();
326                         if (decl.type().spelling().equals("_Bool")) {
327                             // special case boolean constants
328                             yield entry.success(typeMaker.makeType(decl.type()), value == 0L);
329                         } else {
330                             yield entry.success(typeMaker.makeType(decl.type()), value);
331                         }
332                     }
333                     case FloatingPoint -> {
334                         double value = result.getAsFloat();
335                         yield entry.success(typeMaker.makeType(decl.type()), value);
336                     }
337                     case StrLiteral -> {
338                         String value = result.getAsString();
339                         yield entry.success(typeMaker.makeType(decl.type()), value);
340                     }
341                     default -> {
342                         Type type = decl.type().equals(decl.type().canonicalType()) ?
343                                 null : typeMaker.makeType(decl.type());
344                         yield entry.failure(type);
345                     }
346                 };
347                 newEntry.update();
348             }
349         }
350 
351         void reparseMacros(boolean recovery) {
352             String snippet = macroDecl(recovery);
353             TreeMaker treeMaker = new TreeMaker();
354             try {
355                 reparser.reparse(snippet)
356                         .filter(c -> c.kind() == CursorKind.VarDecl &&
357                                 c.spelling().contains("jextract$"))
358                         .forEach(c -> updateTable(treeMaker.typeMaker, c));
359             } finally {
360                 treeMaker.typeMaker.resolveTypeReferences();
361             }
362         }
363 
364         String macroDecl(boolean recovery) {
365             StringBuilder buf = new StringBuilder();
366             if (recovery) {
367                 buf.append("#include <stdint.h>\n");
368             }
369             macrosByMangledName.values().stream()
370                     .filter(e -> !e.isSuccess()) // skip macros that already have passed
371                     .filter(recovery ? Entry::isRecoverableFailure : Entry::isUnparsed)
372                     .forEach(e -> {
373                         buf.append("__auto_type ")
374                                 .append(e.mangledName())
375                                 .append(" = ");
376                         if (recovery) {
377                             buf.append("(uintptr_t)");
378                         }
379                         buf.append(e.name)
380                                 .append(";\n");
381                     });
382             return buf.toString();
383         }
384     }
385 }