diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java index 5948f01fd..74fea04e4 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java @@ -148,7 +148,11 @@ public void run(Path scratchPath) { } try { - var prInstance = new PullRequestInstance(scratchPath.resolve("pr"), pr, bot.ignoreStaleReviews()); + var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds")); + var prInstance = new PullRequestInstance(scratchPath.resolve("pr"), + new HostedRepositoryPool(seedPath), + pr, + bot.ignoreStaleReviews()); CheckRun.execute(this, pr, prInstance, comments, allReviews, activeReviews, labels, census); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/IntegrateCommand.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/IntegrateCommand.java index 38b633986..f5b6ac0c8 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/IntegrateCommand.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/IntegrateCommand.java @@ -78,7 +78,11 @@ public void handle(PullRequestBot bot, PullRequest pr, CensusInstance censusInst var sanitizedUrl = URLEncoder.encode(pr.repository().webUrl().toString(), StandardCharsets.UTF_8); var path = scratchPath.resolve("pr.integrate").resolve(sanitizedUrl); - var prInstance = new PullRequestInstance(path, pr, bot.ignoreStaleReviews()); + var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds")); + var prInstance = new PullRequestInstance(path, + new HostedRepositoryPool(seedPath), + pr, + bot.ignoreStaleReviews()); var localHash = prInstance.commit(censusInstance.namespace(), censusInstance.configuration().census().domain(), null); // Validate the target hash if requested diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/LabelerWorkItem.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/LabelerWorkItem.java index 682fd73c5..0b441136d 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/LabelerWorkItem.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/LabelerWorkItem.java @@ -22,7 +22,7 @@ */ package org.openjdk.skara.bots.pr; -import org.openjdk.skara.forge.PullRequest; +import org.openjdk.skara.forge.*; import java.io.*; import java.nio.file.Path; @@ -63,7 +63,11 @@ public void run(Path scratchPath) { return; } try { - var prInstance = new PullRequestInstance(scratchPath.resolve("labeler"), pr, bot.ignoreStaleReviews()); + var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds")); + var prInstance = new PullRequestInstance(scratchPath.resolve("labeler"), + new HostedRepositoryPool(seedPath), + pr, + bot.ignoreStaleReviews()); var newLabels = getLabels(prInstance); var currentLabels = pr.labels().stream() .filter(key -> bot.labelPatterns().containsKey(key)) diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBot.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBot.java index 0c34fa254..31bcd32ef 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBot.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBot.java @@ -28,6 +28,7 @@ import org.openjdk.skara.json.JSONValue; import org.openjdk.skara.vcs.Hash; +import java.nio.file.Path; import java.util.*; import java.util.concurrent.*; import java.util.logging.Logger; @@ -45,6 +46,7 @@ class PullRequestBot implements Bot { private final IssueProject issueProject; private final boolean ignoreStaleReviews; private final Pattern allowedTargetBranches; + private final Path seedStorage; private final ConcurrentMap currentLabels; private final PullRequestUpdateCache updateCache; private final Logger log = Logger.getLogger("org.openjdk.skara.bots.pr"); @@ -53,7 +55,7 @@ class PullRequestBot implements Bot { Map> labelPatterns, Map externalCommands, Map blockingLabels, Set readyLabels, Map readyComments, IssueProject issueProject, boolean ignoreStaleReviews, - Pattern allowedTargetBranches) { + Pattern allowedTargetBranches, Path seedStorage) { remoteRepo = repo; this.censusRepo = censusRepo; this.censusRef = censusRef; @@ -65,6 +67,7 @@ class PullRequestBot implements Bot { this.readyComments = readyComments; this.ignoreStaleReviews = ignoreStaleReviews; this.allowedTargetBranches = allowedTargetBranches; + this.seedStorage = seedStorage; this.currentLabels = new ConcurrentHashMap<>(); this.updateCache = new PullRequestUpdateCache(); @@ -180,4 +183,8 @@ boolean ignoreStaleReviews() { Pattern allowedTargetBranches() { return allowedTargetBranches; } + + Optional seedStorage() { + return Optional.ofNullable(seedStorage); + } } diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotBuilder.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotBuilder.java index ae9c56ccc..7f9941cc9 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotBuilder.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotBuilder.java @@ -25,6 +25,7 @@ import org.openjdk.skara.forge.HostedRepository; import org.openjdk.skara.issuetracker.IssueProject; +import java.nio.file.Path; import java.util.*; import java.util.regex.Pattern; @@ -40,6 +41,7 @@ public class PullRequestBotBuilder { private IssueProject issueProject = null; private boolean ignoreStaleReviews = false; private Pattern allowedTargetBranches = Pattern.compile(".*"); + private Path seedStorage = null; PullRequestBotBuilder() { } @@ -99,8 +101,14 @@ public PullRequestBotBuilder allowedTargetBranches(String allowedTargetBranches) return this; } + public PullRequestBotBuilder seedStorage(Path seedStorage) { + this.seedStorage = seedStorage; + return this; + } + public PullRequestBot build() { return new PullRequestBot(repo, censusRepo, censusRef, labelPatterns, externalCommands, blockingLabels, - readyLabels, readyComments, issueProject, ignoreStaleReviews, allowedTargetBranches); + readyLabels, readyComments, issueProject, ignoreStaleReviews, allowedTargetBranches, + seedStorage); } } \ No newline at end of file diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotFactory.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotFactory.java index c3ffcb441..f185a0be4 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotFactory.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestBotFactory.java @@ -73,7 +73,8 @@ public List create(BotConfiguration configuration) { .blockingLabels(blockers) .readyLabels(readyLabels) .readyComments(readyComments) - .externalCommands(external); + .externalCommands(external) + .seedStorage(configuration.storageFolder().resolve("seeds")); if (repo.value().contains("labels")) { var labelPatterns = new HashMap>(); diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java index a3fbef4f2..50c46586c 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java @@ -43,16 +43,17 @@ class PullRequestInstance { private final Hash baseHash; private final boolean ignoreStaleReviews; - PullRequestInstance(Path localRepoPath, PullRequest pr, boolean ignoreStaleReviews) throws IOException { + PullRequestInstance(Path localRepoPath, HostedRepositoryPool hostedRepositoryPool, PullRequest pr, boolean ignoreStaleReviews) throws IOException { this.pr = pr; this.ignoreStaleReviews = ignoreStaleReviews; + + // Materialize the PR's source and target ref var repository = pr.repository(); + localRepo = hostedRepositoryPool.checkout(pr, localRepoPath); + localRepo.fetch(repository.url(), "+" + pr.targetRef() + ":pr_prinstance"); - // Materialize the PR's target ref - localRepo = Repository.materialize(localRepoPath, repository.url(), - "+" + pr.targetRef() + ":pr_prinstance_" + repository.name()); - targetHash = localRepo.fetch(repository.url(), pr.targetRef()); - headHash = localRepo.fetch(repository.url(), pr.headHash().hex()); + targetHash = pr.targetHash(); + headHash = pr.headHash(); baseHash = localRepo.mergeBase(targetHash, headHash); } diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java index 9627f7c6f..7bd32d3f8 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java @@ -22,7 +22,7 @@ */ package org.openjdk.skara.bots.pr; -import org.openjdk.skara.forge.PullRequest; +import org.openjdk.skara.forge.*; import org.openjdk.skara.issuetracker.Comment; import org.openjdk.skara.vcs.Hash; @@ -73,7 +73,11 @@ public void handle(PullRequestBot bot, PullRequest pr, CensusInstance censusInst var sanitizedUrl = URLEncoder.encode(pr.repository().webUrl().toString(), StandardCharsets.UTF_8); var path = scratchPath.resolve("pr.sponsor").resolve(sanitizedUrl); - var prInstance = new PullRequestInstance(path, pr, bot.ignoreStaleReviews()); + var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds")); + var prInstance = new PullRequestInstance(path, + new HostedRepositoryPool(seedPath), + pr, + bot.ignoreStaleReviews()); var localHash = prInstance.commit(censusInstance.namespace(), censusInstance.configuration().census().domain(), comment.author().id()); diff --git a/forge/src/main/java/org/openjdk/skara/forge/HostedRepositoryPool.java b/forge/src/main/java/org/openjdk/skara/forge/HostedRepositoryPool.java new file mode 100644 index 000000000..3b505bf5c --- /dev/null +++ b/forge/src/main/java/org/openjdk/skara/forge/HostedRepositoryPool.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.forge; + +import org.openjdk.skara.vcs.*; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.logging.Logger; + +public class HostedRepositoryPool { + private final Path seedStorage; + private final Logger log = Logger.getLogger("org.openjdk.skara.forge"); + + public HostedRepositoryPool(Path seedStorage) { + this.seedStorage = seedStorage; + } + + private class HostedRepositoryInstance { + private final HostedRepository hostedRepository; + private final Path seed; + private final String ref; + + private HostedRepositoryInstance(HostedRepository hostedRepository, String ref) { + this.hostedRepository = hostedRepository; + this.seed = seedStorage.resolve(hostedRepository.name()); + this.ref = ref; + } + + private class NewClone { + private final Repository repository; + private final Hash fetchHead; + + NewClone(Repository repository, Hash fetchHead) { + this.repository = repository; + this.fetchHead = fetchHead; + } + + Repository repository() { + return repository; + } + + Hash fetchHead() { + return fetchHead; + } + } + + private void clearDirectory(Path directory) { + try { + Files.walk(directory) + .map(Path::toFile) + .sorted(Comparator.reverseOrder()) + .forEach(File::delete); + } catch (IOException io) { + throw new RuntimeException(io); + } + } + + private void initializeSeed() throws IOException { + if (!Files.exists(seed)) { + Files.createDirectories(seed.getParent()); + var tmpSeedFolder = seed.resolveSibling(seed.getFileName().toString() + "-" + UUID.randomUUID()); + Repository.clone(hostedRepository.url(), tmpSeedFolder, true); + try { + Files.move(tmpSeedFolder, seed); + log.info("Seeded repository " + hostedRepository.name() + " into " + seed); + } catch (IOException e) { + log.info("Failed to populate seed folder " + seed + " - perhaps due to a benign race. Ignoring.."); + clearDirectory(tmpSeedFolder); + } + } + } + + private Repository cloneSeeded(Path path) throws IOException { + initializeSeed(); + log.info("Using seed folder " + seed + " when cloning into " + path); + return Repository.clone(hostedRepository.url(), path, false, seed); + } + + private NewClone fetchRef(Repository repository) throws IOException { + var fetchHead = repository.fetch(hostedRepository.url(), "+" + ref + ":" + ref); + return new NewClone(repository, fetchHead); + } + + private NewClone materializeClone(Path path) throws IOException { + var localRepo = Repository.get(path); + if (localRepo.isEmpty()) { + return fetchRef(cloneSeeded(path)); + } else { + var localRepoInstance = localRepo.get(); + if (!localRepoInstance.isHealthy()) { + var preserveUnhealthy = seed.resolveSibling(seed.getFileName().toString() + "-unhealthy-" + UUID.randomUUID()); + log.severe("Unhealthy local repository detected - preserved in: " + preserveUnhealthy); + Files.move(localRepoInstance.root(), preserveUnhealthy); + return fetchRef(cloneSeeded(path)); + } else { + try { + localRepoInstance.clean(); + return fetchRef(localRepoInstance); + } catch (IOException e) { + var preserveUnclean = seed.resolveSibling(seed.getFileName().toString() + "-unclean-" + UUID.randomUUID()); + log.severe("Uncleanable local repository detected - preserved in: " + preserveUnclean); + Files.move(localRepoInstance.root(), preserveUnclean); + return fetchRef(cloneSeeded(path)); + } + } + } + } + } + + public Repository materialize(HostedRepository hostedRepository, String ref, Path path) throws IOException { + var hostedRepositoryInstance = new HostedRepositoryInstance(hostedRepository, ref); + var clone = hostedRepositoryInstance.materializeClone(path); + return clone.repository(); + } + + public Repository materialize(PullRequest pr, Path path) throws IOException { + return materialize(pr.repository(), pr.sourceRef(), path); + } + + public Repository checkout(HostedRepository hostedRepository, String ref, Path path) throws IOException { + var hostedRepositoryInstance = new HostedRepositoryInstance(hostedRepository, ref); + var clone = hostedRepositoryInstance.materializeClone(path); + var localRepo = clone.repository(); + try { + localRepo.checkout(clone.fetchHead(), true); + } catch (IOException e) { + var preserveUnchecked = hostedRepositoryInstance.seed.resolveSibling( + hostedRepositoryInstance.seed.getFileName().toString() + "-unchecked-" + UUID.randomUUID()); + log.severe("Uncheckoutable local repository detected - preserved in: " + preserveUnchecked); + Files.move(localRepo.root(), preserveUnchecked); + clone = hostedRepositoryInstance.fetchRef(hostedRepositoryInstance.cloneSeeded(path)); + localRepo = clone.repository(); + localRepo.checkout(clone.fetchHead(), true); + } + return localRepo; + } + + public Repository checkout(PullRequest pr, Path path) throws IOException { + return checkout(pr.repository(), pr.sourceRef(), path); + } +} diff --git a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java index 70aa06925..51136ae04 100644 --- a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java +++ b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java @@ -60,6 +60,11 @@ public List branches() throws IOException { return branches; } + @Override + public List branches(String remote) throws IOException { + return branches; + } + void setBranches(List branches) { this.branches = branches; } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java b/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java index 339811ffc..c63a7f44d 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Path; -import java.nio.file.Files; import java.time.ZonedDateTime; import java.util.*; @@ -194,13 +193,17 @@ static Repository clone(URI from, Path to) throws IOException { } static Repository clone(URI from, Path to, boolean isBare) throws IOException { - return from.getPath().toString().endsWith(".git") ? - GitRepository.clone(from, to, isBare) : HgRepository.clone(from, to, isBare); + return clone(from, to, isBare, null); + } + + static Repository clone(URI from, Path to, boolean isBare, Path seed) throws IOException { + return from.getPath().endsWith(".git") ? + GitRepository.clone(from, to, isBare, seed) : HgRepository.clone(from, to, isBare, seed); } static Repository mirror(URI from, Path to) throws IOException { return from.getPath().toString().endsWith(".git") ? GitRepository.mirror(from, to) : - HgRepository.clone(from, to, true); // hg does not have concept of "mirror" + HgRepository.clone(from, to, true, null); // hg does not have concept of "mirror" } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java index ca354dbbd..b4f9b019b 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java @@ -1050,7 +1050,7 @@ public Optional upstreamFor(Branch b) throws IOException { } } - public static Repository clone(URI from, Path to, boolean isBare) throws IOException { + public static Repository clone(URI from, Path to, boolean isBare, Path seed) throws IOException { var cmd = new ArrayList(); cmd.addAll(List.of("git", "clone")); if (isBare) { @@ -1058,6 +1058,10 @@ public static Repository clone(URI from, Path to, boolean isBare) throws IOExcep } else { cmd.add("--recurse-submodules"); } + if (seed != null) { + cmd.add("--reference-if-able"); + cmd.add(seed.toString()); + } cmd.addAll(List.of(from.toString(), to.toString())); try (var p = capture(Path.of("").toAbsolutePath(), cmd)) { await(p); diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java index 9972ba224..a613a0f54 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java @@ -1097,7 +1097,7 @@ public Optional upstreamFor(Branch b) throws IOException { return Optional.of(b.name()); } - public static Repository clone(URI from, Path to, boolean isBare) throws IOException { + public static Repository clone(URI from, Path to, boolean isBare, Path seed) throws IOException { var cmd = new ArrayList(); cmd.addAll(List.of("hg", "clone")); if (isBare) { @@ -1186,7 +1186,7 @@ public List remotes() throws IOException { @Override public void addSubmodule(String pullPath, Path path) throws IOException { var uri = Files.exists(Path.of(pullPath)) ? Path.of(pullPath).toUri().toString() : pullPath; - HgRepository.clone(URI.create(uri), root().resolve(path).toAbsolutePath(), false); + HgRepository.clone(URI.create(uri), root().resolve(path).toAbsolutePath(), false, null); var hgSub = root().resolve(".hgsub"); Files.writeString(hgSub, path.toString() + " = " + pullPath + "\n", StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE);