1 /*
  2  *  Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
  3  *  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  *  This code is free software; you can redistribute it and/or modify it
  6  *  under the terms of the GNU General Public License version 2 only, as
  7  *  published by the Free Software Foundation.  Oracle designates this
  8  *  particular file as subject to the "Classpath" exception as provided
  9  *  by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  *  This code is distributed in the hope that it will be useful, but WITHOUT
 12  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  *  version 2 for more details (a copy is included in the LICENSE file that
 15  *  accompanied this code).
 16  *
 17  *  You should have received a copy of the GNU General Public License version
 18  *  2 along with this work; if not, write to the Free Software Foundation,
 19  *  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  *   Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  *  or visit www.oracle.com if you need additional information or have any
 23  *  questions.
 24  *
 25  */
 26 
 27 package jdk.internal.jextract.impl;
 28 
 29 import jdk.incubator.jextract.Declaration;
 30 import jdk.incubator.jextract.Position;
 31 import jdk.incubator.jextract.Type;
 32 import jdk.incubator.jextract.JextractTool;
 33 import jdk.internal.clang.Cursor;
 34 import jdk.internal.clang.CursorKind;
 35 import jdk.internal.clang.Diagnostic;
 36 import jdk.internal.clang.EvalResult;
 37 import jdk.internal.clang.Index;
 38 import jdk.internal.clang.LibClang;
 39 import jdk.internal.clang.TranslationUnit;
 40 
 41 import java.io.IOException;
 42 import java.nio.file.Files;
 43 import java.nio.file.Path;
 44 import java.util.Collection;
 45 import java.util.LinkedHashMap;
 46 import java.util.List;
 47 import java.util.Map;
 48 import java.util.Optional;
 49 import java.util.stream.Collectors;
 50 import java.util.stream.Stream;
 51 
 52 class MacroParserImpl {
 53 
 54     private final ClangReparser reparser;
 55     private final TreeMaker treeMaker;
 56     final MacroTable macroTable;
 57 
 58     private MacroParserImpl(ClangReparser reparser, TreeMaker treeMaker) {
 59         this.reparser = reparser;
 60         this.treeMaker = treeMaker;
 61         this.macroTable = new MacroTable();
 62     }
 63 
 64     static MacroParserImpl make(TreeMaker treeMaker, TranslationUnit tu, Collection<String> args) {
 65         ClangReparser reparser;
 66         try {
 67             reparser = new ClangReparser(tu, args);
 68         } catch (IOException | Index.ParsingFailedException ex) {
 69             throw new RuntimeException(ex);
 70         }
 71 
 72         return new MacroParserImpl(reparser, treeMaker);
 73     }
 74 
 75     /**
 76      * This method attempts to evaluate the macro. Evaluation occurs in two steps: first, an attempt is made
 77      * to see if the macro corresponds to a simple numeric constant. If so, the constant is parsed in Java directly.
 78      * If that is not possible (e.g. because the macro refers to other macro, or has a more complex grammar), fall
 79      * back to use clang evaluation support.
 80      */
 81     Optional<Declaration.Constant> parseConstant(Position pos, String name, String[] tokens) {
 82         if (!(pos instanceof TreeMaker.CursorPosition)) {
 83             return Optional.empty();
 84         } else {
 85             Cursor cursor = ((TreeMaker.CursorPosition)pos).cursor();
 86             if (cursor.isMacroFunctionLike()) {
 87                 return Optional.empty();
 88             } else if (tokens.length == 2) {
 89                 //check for fast path
 90                 Integer num = toNumber(tokens[1]);
 91                 if (num != null) {
 92                     return Optional.of(treeMaker.createMacro(cursor, name, Type.primitive(Type.Primitive.Kind.Int), (long)num));
 93                 }
 94             }
 95             macroTable.enterMacro(name, tokens, cursor);
 96             return Optional.empty();
 97         }
 98     }
 99 
100     private Integer toNumber(String str) {
101         try {
102             // Integer.decode supports '#' hex literals which is not valid in C.
103             return str.length() > 0 && str.charAt(0) != '#'? Integer.decode(str) : null;
104         } catch (NumberFormatException nfe) {
105             return null;
106         }
107     }
108 
109     /**
110      * This class allows client to reparse a snippet of code against a given set of include files.
111      * For performance reasons, the set of includes (which comes from the jextract parser) is compiled
112      * into a precompiled header, so as to speed to incremental recompilation of the generated snippets.
113      */
114     static class ClangReparser {
115         final Path macro;
116         final Index macroIndex = LibClang.createIndex(true);
117         final TranslationUnit macroUnit;
118 
119         public ClangReparser(TranslationUnit tu, Collection<String> args) throws IOException, Index.ParsingFailedException {
120             Path precompiled = Files.createTempFile("jextract$", ".pch");
121             precompiled.toFile().deleteOnExit();
122             tu.save(precompiled);
123             this.macro = Files.createTempFile("jextract$", ".h");
124             this.macro.toFile().deleteOnExit();
125             String[] patchedArgs = Stream.concat(
126                 Stream.of(
127                     // Avoid system search path, use bundled instead
128                     "-nostdinc",
129                     "-ferror-limit=0",
130                     // precompiled header
131                     "-include-pch", precompiled.toAbsolutePath().toString()),
132                 args.stream()).toArray(String[]::new);
133             this.macroUnit = macroIndex.parse(macro.toAbsolutePath().toString(),
134                     this::processDiagnostics,
135                     false, //add serialization support (needed for macros)
136                     patchedArgs);
137         }
138 
139         void processDiagnostics(Diagnostic diag) {
140             if (JextractTool.DEBUG) {
141                 System.err.println("Error while processing macro: " + diag.spelling());
142             }
143         }
144 
145         public Stream<Cursor> reparse(String snippet) {
146             macroUnit.reparse(this::processDiagnostics,
147                     Index.UnsavedFile.of(macro, snippet));
148             return macroUnit.getCursor().children();
149         }
150     }
151 
152     /**
153      * This abstraction is used to collect all macros which could not be interpreted during {@link #parseConstant(Position, String, String[])}.
154      * All unparsed macros in the table can have three different states: UNPARSED (which means the macro has not been parsed yet),
155      * SUCCESS (which means the macro has been parsed and has a type and a value) and FAILURE, which means the macro has been
156      * parsed with some errors, but for which we were at least able to infer a type.
157      *
158      * The reparsing process goes as follows:
159      * 1. all unparsed macros are added to the table in the UNPARSED state.
160      * 2. a snippet for all macros in the UNPARSED state is compiled and the table state is updated
161      * 3. a recovery snippet for all macros in the FAILURE state is compiled and the table state is updated again
162      * 4. we repeat from (2) until no further progress is made.
163      * 5. we return a list of macro which are in the SUCCESS state.
164      *
165      * State transitions in the table are as follows:
166      * - an UNPARSED macro can go to either SUCCESS, to FAILURE or be removed (if not even a type can be inferred)
167      * - a FAILURE macro can go to either SUCCESS (if recovery step succeds) or be removed
168      * - a SUCCESS macro cannot go in any other state
169      */
170     class MacroTable {
171 
172         final Map<String, Entry> macrosByMangledName = new LinkedHashMap<>();
173 
174         abstract class Entry {
175             final String name;
176             final String[] tokens;
177             final Cursor cursor;
178 
179             Entry(String name, String[] tokens, Cursor cursor) {
180                 this.name = name;
181                 this.tokens = tokens;
182                 this.cursor = cursor;
183             }
184 
185             String mangledName() {
186                 return "jextract$macro$" + name;
187             }
188 
189             Entry success(Type type, Object value) {
190                 throw new IllegalStateException();
191             }
192 
193             Entry failure(Type type) {
194                 throw new IllegalStateException();
195             }
196 
197             boolean isSuccess() {
198                 return false;
199             }
200             boolean isRecoverableFailure() {
201                 return false;
202             }
203             boolean isUnparsed() {
204                 return false;
205             }
206 
207             void update() {
208                 macrosByMangledName.put(mangledName(), this);
209             }
210         }
211 
212         class Unparsed extends Entry {
213             Unparsed(String name, String[] tokens, Cursor cursor) {
214                 super(name, tokens, cursor);
215             }
216 
217             @Override
218             Entry success(Type type, Object value) {
219                 return new Success(name, tokens, cursor, type, value);
220             }
221 
222             @Override
223             Entry failure(Type type) {
224                 return type != null ?
225                         new RecoverableFailure(name, tokens, cursor, type) :
226                         new UnparseableMacro(name, tokens, cursor);
227             }
228 
229             @Override
230             boolean isUnparsed() {
231                 return true;
232             }
233 
234             @Override
235             void update() {
236                 throw new IllegalStateException();
237             }
238         }
239 
240         class RecoverableFailure extends Entry {
241 
242             final Type type;
243 
244             public RecoverableFailure(String name, String[] tokens, Cursor cursor, Type type) {
245                 super(name, tokens, cursor);
246                 this.type = type;
247             }
248 
249             @Override
250             Entry success(Type type, Object value) {
251                 return new Success(name, tokens, cursor, this.type, value);
252             }
253 
254             @Override
255             Entry failure(Type type) {
256                 return new UnparseableMacro(name, tokens, cursor);
257             }
258 
259             @Override
260             boolean isRecoverableFailure() {
261                 return true;
262             }
263         }
264 
265         class Success extends Entry {
266             final Type type;
267             final Object value;
268 
269             public Success(String name, String[] tokens, Cursor cursor, Type type, Object value) {
270                 super(name, tokens, cursor);
271                 this.type = type;
272                 this.value = value;
273             }
274 
275             @Override
276             boolean isSuccess() {
277                 return true;
278             }
279 
280             public Object value() {
281                 return value;
282             }
283         }
284 
285         class UnparseableMacro extends Entry {
286 
287             UnparseableMacro(String name, String[] tokens, Cursor cursor) {
288                 super(name, tokens, cursor);
289             }
290 
291             @Override
292             void update() {
293                 macrosByMangledName.remove(mangledName());
294             }
295         };
296 
297         void enterMacro(String name, String[] tokens, Cursor cursor) {
298             Unparsed unparsed = new Unparsed(name, tokens, cursor);
299             macrosByMangledName.put(unparsed.mangledName(), unparsed);
300         }
301 
302         public List<Declaration.Constant> reparseConstants() {
303             int last = -1;
304             while (macrosByMangledName.size() > 0 && last != macrosByMangledName.size()) {
305                 last = macrosByMangledName.size();
306                 // step 1 - try parsing macros as var declarations
307                 reparseMacros(false);
308                 // step 2 - retry failed parsed macros as pointers
309                 reparseMacros(true);
310             }
311             treeMaker.typeMaker.resolveTypeReferences();
312             return macrosByMangledName.values().stream()
313                     .filter(Entry::isSuccess)
314                     .map(e -> treeMaker.createMacro(e.cursor, e.name, ((Success)e).type, ((Success)e).value))
315                     .collect(Collectors.toList());
316         }
317 
318         void updateTable(TypeMaker typeMaker, Cursor decl) {
319             String mangledName = decl.spelling();
320             Entry entry = macrosByMangledName.get(mangledName);
321             try (EvalResult result = decl.eval()) {
322                 Entry newEntry = switch (result.getKind()) {
323                     case Integral -> {
324                         long value = result.getAsInt();
325                         yield entry.success(typeMaker.makeType(decl.type()), value);
326                     }
327                     case FloatingPoint -> {
328                         double value = result.getAsFloat();
329                         yield entry.success(typeMaker.makeType(decl.type()), value);
330                     }
331                     case StrLiteral -> {
332                         String value = result.getAsString();
333                         yield entry.success(typeMaker.makeType(decl.type()), value);
334                     }
335                     default -> {
336                         Type type = decl.type().equals(decl.type().canonicalType()) ?
337                                 null : typeMaker.makeType(decl.type());
338                         yield entry.failure(type);
339                     }
340                 };
341                 newEntry.update();
342             }
343         }
344 
345         void reparseMacros(boolean recovery) {
346             String snippet = macroDecl(recovery);
347             TreeMaker treeMaker = new TreeMaker();
348             try {
349                 reparser.reparse(snippet)
350                         .filter(c -> c.kind() == CursorKind.VarDecl &&
351                                 c.spelling().contains("jextract$"))
352                         .forEach(c -> updateTable(treeMaker.typeMaker, c));
353             } finally {
354                 treeMaker.typeMaker.resolveTypeReferences();
355             }
356         }
357 
358         String macroDecl(boolean recovery) {
359             StringBuilder buf = new StringBuilder();
360             if (recovery) {
361                 buf.append("#include <stdint.h>\n");
362             }
363             macrosByMangledName.values().stream()
364                     .filter(e -> !e.isSuccess()) // skip macros that already have passed
365                     .filter(recovery ? Entry::isRecoverableFailure : Entry::isUnparsed)
366                     .forEach(e -> {
367                         buf.append("__auto_type ")
368                                 .append(e.mangledName())
369                                 .append(" = ");
370                         if (recovery) {
371                             buf.append("(uintptr_t)");
372                         }
373                         buf.append(e.name)
374                                 .append(";\n");
375                     });
376             return buf.toString();
377         }
378     }
379 }