javaparser/utils/SourceRoot.java

package com.github.javaparser.utils;

import com.github.javaparser.JavaParser;
import com.github.javaparser.ParseProblemException;
import com.github.javaparser.ParseResult;
import com.github.javaparser.ParserConfiguration;
import com.github.javaparser.ast.CompilationUnit;
import com.github.javaparser.printer.PrettyPrinter;

import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveAction;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static com.github.javaparser.ParseStart.COMPILATION_UNIT;
import static com.github.javaparser.Providers.provider;
import static com.github.javaparser.utils.CodeGenerationUtils.fileInPackageRelativePath;
import static com.github.javaparser.utils.CodeGenerationUtils.packageAbsolutePath;
import static com.github.javaparser.utils.SourceRoot.Callback.Result.SAVE;
import static com.github.javaparser.utils.Utils.assertNotNull;
import static java.nio.file.FileVisitResult.CONTINUE;
import static java.nio.file.FileVisitResult.SKIP_SUBTREE;

/**
 * A collection of Java source files located in one directory and its subdirectories on the file system. Files can be
 * parsed and written back one by one or all together. <b>Note that</b> the internal cache used is thread-safe.
 * <ul>
 * <li>methods called "tryToParse..." will return their result inside a "ParseResult", which supports parse successes and failures.</li>
 * <li>methods called "parse..." will return "CompilationUnit"s. If a file fails to parse, an exception is thrown.</li>
 * <li>methods ending in "...Parallelized" will speed up parsing by using multiple threads.</li>
 * </ul>
 */
public class SourceRoot {
    @FunctionalInterface
    public interface Callback {
        enum Result {
            SAVE, DONT_SAVE
        }

        /**
         * @param localPath the path to the file that was parsed, relative to the source root path.
         * @param absolutePath the absolute path to the file that was parsed.
         * @param result the result of of parsing the file.
         */
        Result process(Path localPath, Path absolutePath, ParseResult<CompilationUnit> result);
    }

    private final Path root;
    private final Map<Path, ParseResult<CompilationUnit>> cache = new ConcurrentHashMap<>();
    private ParserConfiguration parserConfiguration = new ParserConfiguration();
    private Function<CompilationUnit, String> printer = new PrettyPrinter()::print;
    private static final Pattern JAVA_IDENTIFIER = Pattern.compile("\\p{javaJavaIdentifierStart}\\p{javaJavaIdentifierPart}*");

    public SourceRoot(Path root) {
        assertNotNull(root);
        if (!Files.isDirectory(root)) {
            throw new IllegalArgumentException("Only directories are allowed as root path!");
        }
        this.root = root.normalize();
        Log.info("New source root at \"%s\"", this.root);
    }

    public SourceRoot(Path root, ParserConfiguration parserConfiguration) {
        this(root);
        setParserConfiguration(parserConfiguration);
    }

    /**
     * Tries to parse a .java files under the source root and returns the ParseResult. It keeps track of the parsed file
     * so you can write it out with the saveAll() call. Note that the cache grows with every file parsed, so if you
     * don't need saveAll(), or you don't ask SourceRoot to parse files multiple times (where the cache is useful) you
     * might want to use the parse method with a callback.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     * @deprecated pass ParserConfiguration instead of JavaParser
     */
    @Deprecated
    public ParseResult<CompilationUnit> tryToParse(String startPackage, String filename, JavaParser javaParser)
            throws IOException {
        return tryToParse(startPackage, filename, javaParser.getParserConfiguration());
    }

    /**
     * Tries to parse a .java files under the source root and returns the ParseResult. It keeps track of the parsed file
     * so you can write it out with the saveAll() call. Note that the cache grows with every file parsed, so if you
     * don't need saveAll(), or you don't ask SourceRoot to parse files multiple times (where the cache is useful) you
     * might want to use the parse method with a callback.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public ParseResult<CompilationUnit> tryToParse(String startPackage, String filename, ParserConfiguration configuration) throws IOException {
        assertNotNull(startPackage);
        assertNotNull(filename);
        final Path relativePath = fileInPackageRelativePath(startPackage, filename);
        if (cache.containsKey(relativePath)) {
            Log.trace("Retrieving cached %s", relativePath);
            return cache.get(relativePath);
        }
        final Path path = root.resolve(relativePath);
        Log.trace("Parsing %s", path);
        final ParseResult<CompilationUnit> result = new JavaParser(configuration)
                .parse(COMPILATION_UNIT, provider(path));
        result.getResult().ifPresent(cu -> cu.setStorage(path));
        cache.put(relativePath, result);
        return result;
    }

    /**
     * Tries to parse a .java files under the source root and returns the ParseResult. It keeps track of the parsed file
     * so you can write it out with the saveAll() call. Note that the cache grows with every file parsed, so if you
     * don't need saveAll(), or you don't ask SourceRoot to parse files multiple times (where the cache is useful) you
     * might want to use the parse method with a callback.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public ParseResult<CompilationUnit> tryToParse(String startPackage, String filename) throws IOException {
        return tryToParse(startPackage, filename, parserConfiguration);
    }

    /**
     * Tries to parse all .java files in a package recursively, and returns all files ever parsed with this source root.
     * It keeps track of all parsed files so you can write them out with a single saveAll() call. Note that the cache
     * grows with every file parsed, so if you don't need saveAll(), or you don't ask SourceRoot to parse files multiple
     * times (where the cache is useful) you might want to use the parse method with a callback.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public List<ParseResult<CompilationUnit>> tryToParse(String startPackage) throws IOException {
        assertNotNull(startPackage);
        logPackage(startPackage);
        final Path path = packageAbsolutePath(root, startPackage);
        Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                if (!attrs.isDirectory() && file.toString().endsWith(".java")) {
                    Path relative = root.relativize(file.getParent());
                    tryToParse(relative.toString(), file.getFileName().toString());
                }
                return CONTINUE;
            }

            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return isSensibleDirectoryToEnter(dir) ? CONTINUE : SKIP_SUBTREE;
            }
        });
        return getCache();
    }

    private static boolean isSensibleDirectoryToEnter(Path dir) throws IOException {
        final String dirToEnter = dir.getFileName().toString();
        final boolean directoryIsAValidJavaIdentifier = JAVA_IDENTIFIER.matcher(dirToEnter).matches();
        if (Files.isHidden(dir) || !directoryIsAValidJavaIdentifier) {
            Log.trace("Not processing directory \"%s\"", dirToEnter);
            return false;
        }
        return true;
    }

    /**
     * Tries to parse all .java files under the source root recursively, and returns all files ever parsed with this
     * source root. It keeps track of all parsed files so you can write them out with a single saveAll() call. Note that
     * the cache grows with every file parsed, so if you don't need saveAll(), or you don't ask SourceRoot to parse
     * files multiple times (where the cache is useful) you might want to use the parse method with a callback.
     */
    public List<ParseResult<CompilationUnit>> tryToParse() throws IOException {
        return tryToParse("");
    }

    /**
     * Tries to parse all .java files in a package recursively using multiple threads, and returns all files ever parsed
     * with this source root. A new thread is forked each time a new directory is visited and is responsible for parsing
     * all .java files in that directory. <b>Note that</b> to ensure thread safety, a new parser instance is created for
     * every file with the internal parser's (i.e. {@link #setJavaParser}) configuration. It keeps track of all parsed
     * files so you can write them out with a single saveAll() call. Note that the cache grows with every file parsed,
     * so if you don't need saveAll(), or you don't ask SourceRoot to parse files multiple times (where the cache is
     * useful) you might want to use the parse method with a callback.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public List<ParseResult<CompilationUnit>> tryToParseParallelized(String startPackage) {
        assertNotNull(startPackage);
        logPackage(startPackage);
        final Path path = packageAbsolutePath(root, startPackage);
        ParallelParse parse = new ParallelParse(path, (file, attrs) -> {
            if (!attrs.isDirectory() && file.toString().endsWith(".java")) {
                Path relative = root.relativize(file.getParent());
                try {
                    tryToParse(
                            relative.toString(),
                            file.getFileName().toString(),
                            parserConfiguration);
                } catch (IOException e) {
                    Log.error(e);
                }
            }
            return CONTINUE;
        });
        ForkJoinPool pool = new ForkJoinPool();
        pool.invoke(parse);
        return getCache();
    }

    /**
     * Tries to parse all .java files under the source root recursively using multiple threads, and returns all files
     * ever parsed with this source root. A new thread is forked each time a new directory is visited and is responsible
     * for parsing all .java files in that directory. <b>Note that</b> to ensure thread safety, a new parser instance is
     * created for every file with the internal parser's (i.e. {@link #setJavaParser}) configuration. It keeps track of
     * all parsed files so you can write them out with a single saveAll() call. Note that the cache grows with every
     * file parsed, so if you don't need saveAll(), or you don't ask SourceRoot to parse files multiple times (where the
     * cache is useful) you might want to use the parse method with a callback.
     */
    public List<ParseResult<CompilationUnit>> tryToParseParallelized() throws IOException {
        return tryToParseParallelized("");
    }

    /**
     * Parses a .java files under the source root and returns its CompilationUnit. It keeps track of the parsed file so
     * you can write it out with the saveAll() call. Note that the cache grows with every file parsed, so if you don't
     * need saveAll(), or you don't ask SourceRoot to parse files multiple times (where the cache is useful) you might
     * want to use the parse method with a callback.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     * @throws ParseProblemException when something went wrong.
     */
    public CompilationUnit parse(String startPackage, String filename) {
        assertNotNull(startPackage);
        assertNotNull(filename);
        try {
            final ParseResult<CompilationUnit> result = tryToParse(startPackage, filename);
            if (result.isSuccessful()) {
                return result.getResult().get();
            }
            throw new ParseProblemException(result.getProblems());
        } catch (IOException e) {
            throw new ParseProblemException(e);
        }
    }

    /**
     * Tries to parse all .java files in a package recursively and passes them one by one to the callback. In comparison
     * to the other parse methods, this is much more memory efficient, but saveAll() won't work.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     * @deprecated pass ParserConfiguration instead of JavaParser
     */
    @Deprecated
    public SourceRoot parse(String startPackage, JavaParser javaParser, Callback callback) throws IOException {
        return parse(startPackage, javaParser.getParserConfiguration(), callback);
    }

    /**
     * Tries to parse all .java files in a package recursively and passes them one by one to the callback. In comparison
     * to the other parse methods, this is much more memory efficient, but saveAll() won't work.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public SourceRoot parse(String startPackage, ParserConfiguration configuration, Callback callback) throws IOException {
        assertNotNull(startPackage);
        assertNotNull(configuration);
        assertNotNull(callback);
        logPackage(startPackage);
        final JavaParser javaParser = new JavaParser(configuration);
        final Path path = packageAbsolutePath(root, startPackage);
        Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path absolutePath, BasicFileAttributes attrs) throws IOException {
                if (!attrs.isDirectory() && absolutePath.toString().endsWith(".java")) {
                    Path localPath = root.relativize(absolutePath);
                    Log.trace("Parsing %s", localPath);
                    final ParseResult<CompilationUnit> result = javaParser.parse(COMPILATION_UNIT,
                            provider(absolutePath));
                    result.getResult().ifPresent(cu -> cu.setStorage(absolutePath));
                    if (callback.process(localPath, absolutePath, result) == SAVE) {
                        if (result.getResult().isPresent()) {
                            save(result.getResult().get(), path);
                        }
                    }
                }
                return CONTINUE;
            }

            @Override
            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                return isSensibleDirectoryToEnter(dir) ? CONTINUE : SKIP_SUBTREE;
            }
        });
        return this;
    }

    private void logPackage(String startPackage) {
        if (startPackage.isEmpty()) {
            return;
        }
        Log.info("Parsing package \"%s\"", startPackage);
    }

    /**
     * Tries to parse all .java files in a package recursively using multiple threads, and passes them one by one to the
     * callback. A new thread is forked each time a new directory is visited and is responsible for parsing all .java
     * files in that directory. <b>Note that</b> the provided {@link Callback} code must be made thread-safe. <b>Note
     * that</b> to ensure thread safety, a new parser instance is created for every file with the provided {@link
     * ParserConfiguration}. In comparison to the other parse methods, this is much more memory efficient, but saveAll()
     * won't work.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public SourceRoot parseParallelized(String startPackage, ParserConfiguration configuration, Callback callback) {
        assertNotNull(startPackage);
        assertNotNull(configuration);
        assertNotNull(callback);
        logPackage(startPackage);
        final Path path = packageAbsolutePath(root, startPackage);
        ParallelParse parse = new ParallelParse(path, (file, attrs) -> {
            if (!attrs.isDirectory() && file.toString().endsWith(".java")) {
                Path localPath = root.relativize(file);
                Log.trace("Parsing %s", localPath);
                try {
                    ParseResult<CompilationUnit> result = new JavaParser(configuration)
                            .parse(COMPILATION_UNIT, provider(file));
                    result.getResult().ifPresent(cu -> cu.setStorage(file));
                    if (callback.process(localPath, file, result) == SAVE) {
                        if (result.getResult().isPresent()) {
                            save(result.getResult().get(), path);
                        }
                    }
                } catch (IOException e) {
                    Log.error(e);
                }
            }
            return CONTINUE;
        });
        ForkJoinPool pool = new ForkJoinPool();
        pool.invoke(parse);
        return this;
    }

    /**
     * Tries to parse all .java files in a package recursively using multiple threads, and passes them one by one to the
     * callback. A new thread is forked each time a new directory is visited and is responsible for parsing all .java
     * files in that directory. <b>Note that</b> the provided {@link Callback} code must be made thread-safe. <b>Note
     * that</b> to ensure thread safety, a new parser instance is created for every file. In comparison to the other
     * parse methods, this is much more memory efficient, but saveAll() won't work.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public SourceRoot parseParallelized(String startPackage, Callback callback) throws IOException {
        return parseParallelized(startPackage, new ParserConfiguration(), callback);
    }

    /**
     * Tries to parse all .java files recursively using multiple threads, and passes them one by one to the callback. A
     * new thread is forked each time a new directory is visited and is responsible for parsing all .java files in that
     * directory. <b>Note that</b> the provided {@link Callback} code must be made thread-safe. <b>Note that</b> to
     * ensure thread safety, a new parser instance is created for every file. In comparison to the other parse methods,
     * this is much more memory efficient, but saveAll() won't work.
     */
    public SourceRoot parseParallelized(Callback callback) throws IOException {
        return parseParallelized("", new ParserConfiguration(), callback);
    }

    /**
     * Add a newly created Java file to the cache of this source root. It will be saved when saveAll is called.
     *
     * @param startPackage files in this package and deeper are parsed. Pass "" to parse all files.
     */
    public SourceRoot add(String startPackage, String filename, CompilationUnit compilationUnit) {
        assertNotNull(startPackage);
        assertNotNull(filename);
        assertNotNull(compilationUnit);
        Log.trace("Adding new file %s.%s", startPackage, filename);
        final Path path = fileInPackageRelativePath(startPackage, filename);
        final ParseResult<CompilationUnit> parseResult = new ParseResult<>(
                compilationUnit,
                new ArrayList<>(),
                null,
                null);
        cache.put(path, parseResult);
        return this;
    }

    /**
     * Add a newly created Java file to the cache of this source root. It will be saved when saveAll is called. It needs
     * to have its path set.
     */
    public SourceRoot add(CompilationUnit compilationUnit) {
        assertNotNull(compilationUnit);
        if (compilationUnit.getStorage().isPresent()) {
            final Path path = compilationUnit.getStorage().get().getPath();
            Log.trace("Adding new file %s", path);
            final ParseResult<CompilationUnit> parseResult = new ParseResult<>(
                    compilationUnit,
                    new ArrayList<>(),
                    null,
                    null);
            cache.put(path, parseResult);
        } else {
            throw new AssertionError("Files added with this method should have their path set.");
        }
        return this;
    }

    /**
     * Save the given compilation unit to the given path.
     */
    private SourceRoot save(CompilationUnit cu, Path path) {
        assertNotNull(cu);
        assertNotNull(path);
        cu.setStorage(path);
        cu.getStorage().get().save(printer);
        return this;
    }

    /**
     * Save all previously parsed files back to a new path.
     */
    public SourceRoot saveAll(Path root) {
        assertNotNull(root);
        Log.info("Saving all files (%s) to %s", cache.size(), root);
        for (Map.Entry<Path, ParseResult<CompilationUnit>> cu : cache.entrySet()) {
            final Path path = root.resolve(cu.getKey());
            if (cu.getValue().getResult().isPresent()) {
                Log.trace("Saving %s", path);
                save(cu.getValue().getResult().get(), path);
            }
        }
        return this;
    }

    /**
     * Save all previously parsed files back to where they were found.
     */
    public SourceRoot saveAll() {
        return saveAll(root);
    }

    /**
     * The Java files that have been parsed by this source root object, or have been added manually.
     */
    public List<ParseResult<CompilationUnit>> getCache() {
        return new ArrayList<>(cache.values());
    }

    /**
     * The CompilationUnits of the Java files that have been parsed succesfully by this source root object, or have been
     * added manually.
     */
    public List<CompilationUnit> getCompilationUnits() {
        return cache.values().stream()
                .filter(ParseResult::isSuccessful)
                .map(p -> p.getResult().get())
                .collect(Collectors.toList());
    }

    /**
     * The path that was passed in the constructor.
     */
    public Path getRoot() {
        return root;
    }

    /**
     * @deprecated store ParserConfiguration now
     */
    @Deprecated
    public JavaParser getJavaParser() {
        return new JavaParser(parserConfiguration);
    }

    /**
     * Set the parser that is used for parsing by default.
     *
     * @deprecated store ParserConfiguration now
     */
    @Deprecated
    public SourceRoot setJavaParser(JavaParser javaParser) {
        assertNotNull(javaParser);
        this.parserConfiguration = javaParser.getParserConfiguration();
        return this;
    }

    public ParserConfiguration getParserConfiguration() {
        return parserConfiguration;
    }

    /**
     * Set the parser configuration that is used for parsing when no configuration is passed to a method.
     */
    public SourceRoot setParserConfiguration(ParserConfiguration parserConfiguration) {
        assertNotNull(parserConfiguration);
        this.parserConfiguration = parserConfiguration;
        return this;
    }

    /**
     * Set the printing function that transforms compilation units into a string to save.
     */
    public SourceRoot setPrinter(Function<CompilationUnit, String> printer) {
        assertNotNull(printer);
        this.printer = printer;
        return this;
    }

    /**
     * Get the printing function.
     */
    public Function<CompilationUnit, String> getPrinter() {
        return printer;
    }

    /**
     * Executes a recursive file tree walk using threads. A new thread is invoked for each new directory discovered
     * during the walk. For each file visited, the user-provided {@link VisitFileCallback} is called with the current
     * path and file attributes. Any shared resources accessed in a {@link VisitFileCallback} should be made
     * thread-safe.
     */
    private static class ParallelParse extends RecursiveAction {

        private static final long serialVersionUID = 1L;
        private final Path path;
        private final VisitFileCallback callback;

        ParallelParse(Path path, VisitFileCallback callback) {
            this.path = path;
            this.callback = callback;
        }

        @Override
        protected void compute() {
            final List<ParallelParse> walks = new ArrayList<>();
            try {
                Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
                    @Override
                    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
                        if (!SourceRoot.isSensibleDirectoryToEnter(dir)) {
                            return SKIP_SUBTREE;
                        }
                        if (!dir.equals(ParallelParse.this.path)) {
                            ParallelParse w = new ParallelParse(dir, callback);
                            w.fork();
                            walks.add(w);
                            return SKIP_SUBTREE;
                        } else {
                            return CONTINUE;
                        }
                    }

                    @Override
                    public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
                        return callback.process(file, attrs);
                    }
                });
            } catch (IOException e) {
                Log.error(e);
            }

            for (ParallelParse w : walks) {
                w.join();
            }
        }

        interface VisitFileCallback {
            FileVisitResult process(Path file, BasicFileAttributes attrs);
        }
    }
}