1 package bldr;
2
3 import static bldr.Bldr.assertExists;
4 import static bldr.Bldr.curl;
5 import static java.io.IO.print;
6 import static java.io.IO.println;
7
8 import java.net.MalformedURLException;
9 import java.net.URI;
10 import java.net.URISyntaxException;
11 import java.net.URL;
12 import java.net.URLEncoder;
13 import java.nio.charset.StandardCharsets;
14 import java.util.ArrayList;
15 import java.util.Comparator;
16 import java.util.HashMap;
17 import java.util.HashSet;
18 import java.util.List;
19 import java.util.Map;
20 import java.util.Optional;
21 import java.util.Set;
22 import java.util.regex.Matcher;
23 import java.util.regex.Pattern;
24 import java.util.stream.Stream;
25 import org.w3c.dom.Element;
26 import org.w3c.dom.Node;
27
28 public class MavenStyleRepository {
29 private final String repoBase = "https://repo1.maven.org/maven2/";
30 private final String searchBase = "https://search.maven.org/solrsearch/";
31 public Bldr.BuildDir dir;
32
33 Bldr.JarFile jarFile(Id id) {
34 return dir.jarFile(id.artifactAndVersion() + ".jar");
35 }
36
37 Bldr.XMLFile pomFile(Id id) {
38 return dir.xmlFile(id.artifactAndVersion() + ".pom");
39 }
40
41 public enum Scope {
42 TEST,
43 COMPILE,
44 PROVIDED,
45 RUNTIME,
46 SYSTEM;
47
48 static Scope of(String name) {
49 return switch (name.toLowerCase()) {
50 case "test" -> TEST;
51 case "compile" -> COMPILE;
52 case "provided" -> PROVIDED;
53 case "runtime" -> RUNTIME;
54 case "system" -> SYSTEM;
55 default -> COMPILE;
56 };
57 }
58 }
59
60 public record GroupAndArtifactId(GroupId groupId, ArtifactId artifactId) {
61
62 public static GroupAndArtifactId of(String groupAndArtifactId) {
63 int idx = groupAndArtifactId.indexOf('/');
64 return of(groupAndArtifactId.substring(0, idx), groupAndArtifactId.substring(idx + 1));
65 }
66
67 public static GroupAndArtifactId of(GroupId groupId, ArtifactId artifactId) {
68 return new GroupAndArtifactId(groupId, artifactId);
69 }
70
71 public static GroupAndArtifactId of(String groupId, String artifactId) {
72 return of(GroupId.of(groupId), ArtifactId.of(artifactId));
73 }
74
75 String location() {
76 return groupId().string().replace('.', '/') + "/" + artifactId().string();
77 }
78
79 @Override
80 public String toString() {
81 return groupId() + "/" + artifactId();
82 }
83 }
84
85 public sealed interface Id permits DependencyId, MetaDataId {
86 MavenStyleRepository mavenStyleRepository();
87
88 GroupAndArtifactId groupAndArtifactId();
89
90 VersionId versionId();
91
92 default String artifactAndVersion() {
93 return groupAndArtifactId().artifactId().string() + '-' + versionId();
94 }
95
96 default String location() {
97 return mavenStyleRepository().repoBase + groupAndArtifactId().location() + "/" + versionId();
98 }
99
100 default URL url(String suffix) {
101 try {
102 return new URI(location() + "/" + artifactAndVersion() + "." + suffix).toURL();
103 } catch (MalformedURLException | URISyntaxException e) {
104 throw new RuntimeException(e);
105 }
106 }
107 }
108
109 public record DependencyId(
110 MavenStyleRepository mavenStyleRepository,
111 GroupAndArtifactId groupAndArtifactId,
112 VersionId versionId,
113 Scope scope,
114 boolean required)
115 implements Id {
116 @Override
117 public String toString() {
118 return groupAndArtifactId().toString()
119 + "/"
120 + versionId()
121 + ":"
122 + scope.toString()
123 + ":"
124 + (required ? "Required" : "Optiona");
125 }
126 }
127
128 public record Pom(MetaDataId metaDataId, XMLNode xmlNode) {
129 Bldr.JarFile getJar() {
130 var jarFile = metaDataId.mavenStyleRepository().jarFile(metaDataId); // ;
131 metaDataId.mavenStyleRepository.queryAndCache(metaDataId.jarURL(), jarFile);
132 return jarFile;
133 }
134
135 String description() {
136 return xmlNode().xpathQueryString("/project/description/text()");
137 }
138
139 Stream<DependencyId> dependencies() {
140 return xmlNode()
141 .nodes(xmlNode.xpath("/project/dependencies/dependency"))
142 .map(node -> new XMLNode((Element) node))
143 .map(
144 dependency ->
145 new DependencyId(
146 metaDataId().mavenStyleRepository(),
147 GroupAndArtifactId.of(
148 GroupId.of(dependency.xpathQueryString("groupId/text()")),
149 ArtifactId.of(dependency.xpathQueryString("artifactId/text()"))),
150 VersionId.of(dependency.xpathQueryString("version/text()")),
151 Scope.of(dependency.xpathQueryString("scope/text()")),
152 !Boolean.parseBoolean(dependency.xpathQueryString("optional/text()"))));
153 }
154
155 Stream<DependencyId> requiredDependencies() {
156 return dependencies().filter(DependencyId::required);
157 }
158 }
159
160 public Optional<Pom> pom(Id id) {
161 return switch (id) {
162 case MetaDataId metaDataId -> {
163 if (metaDataId.versionId() == VersionId.UNSPECIFIED) {
164 // println("what to do when the version is unspecified");
165 yield Optional.empty();
166 }
167 try {
168 yield Optional.of(
169 new Pom(
170 metaDataId,
171 queryAndCache(
172 metaDataId.pomURL(), metaDataId.mavenStyleRepository.pomFile(metaDataId))));
173 } catch (Throwable e) {
174 throw new RuntimeException(e);
175 }
176 }
177 case DependencyId dependencyId -> {
178 if (metaData(
179 id.groupAndArtifactId().groupId().string(),
180 id.groupAndArtifactId().artifactId().string())
181 instanceof Optional<MetaData> optionalMetaData
182 && optionalMetaData.isPresent()) {
183 if (optionalMetaData
184 .get()
185 .metaDataIds()
186 .filter(metaDataId -> metaDataId.versionId().equals(id.versionId()))
187 .findFirst()
188 instanceof Optional<MetaDataId> metaId
189 && metaId.isPresent()) {
190 yield pom(metaId.get());
191 } else {
192 yield Optional.empty();
193 }
194 } else {
195 yield Optional.empty();
196 }
197 }
198 default -> throw new IllegalStateException("Unexpected value: " + id);
199 };
200 }
201
202 public Optional<Pom> pom(GroupAndArtifactId groupAndArtifactId) {
203 var metaData = metaData(groupAndArtifactId).orElseThrow();
204 var metaDataId = metaData.latestMetaDataId().orElseThrow();
205 return pom(metaDataId);
206 }
207
208 record IdVersions(GroupAndArtifactId groupAndArtifactId, Set<Id> versions) {
209 static IdVersions of(GroupAndArtifactId groupAndArtifactId) {
210 return new IdVersions(groupAndArtifactId, new HashSet<>());
211 }
212 }
213
214
215
216 public static class Dag implements Bldr.ClassPathEntryProvider {
217 private final MavenStyleRepository repo;
218 private final List<GroupAndArtifactId> rootGroupAndArtifactIds;
219 Map<GroupAndArtifactId, IdVersions> nodes = new HashMap<>();
220 Map<IdVersions, List<IdVersions>> edges = new HashMap<>();
221
222 Dag add(Id from, Id to) {
223 var fromNode =
224 nodes.computeIfAbsent(
225 from.groupAndArtifactId(), _ -> IdVersions.of(from.groupAndArtifactId()));
226 fromNode.versions().add(from);
227 var toNode =
228 nodes.computeIfAbsent(
229 to.groupAndArtifactId(), _ -> IdVersions.of(to.groupAndArtifactId()));
230 toNode.versions().add(to);
231 edges.computeIfAbsent(fromNode, k -> new ArrayList<>()).add(toNode);
232 return this;
233 }
234
235 void removeUNSPECIFIED() {
236 nodes
237 .values()
238 .forEach(
239 idversions -> {
240 if (idversions.versions().size() > 1) {
241 List<Id> versions = new ArrayList<>(idversions.versions());
242 idversions.versions().clear();
243 idversions
244 .versions()
245 .addAll(
246 versions.stream()
247 .filter(v -> !v.versionId().equals(VersionId.UNSPECIFIED))
248 .toList());
249 println(idversions);
250 }
251 if (idversions.versions().size() > 1) {
252 throw new IllegalStateException("more than one version");
253 }
254 });
255 }
256
257 Dag(MavenStyleRepository repo, List<GroupAndArtifactId> rootGroupAndArtifactIds) {
258 this.repo = repo;
259 this.rootGroupAndArtifactIds = rootGroupAndArtifactIds;
260
261 Set<Id> unresolved = new HashSet<>();
262 rootGroupAndArtifactIds.forEach(
263 rootGroupAndArtifactId -> {
264 var metaData = repo.metaData(rootGroupAndArtifactId).orElseThrow();
265 var metaDataId = metaData.latestMetaDataId().orElseThrow();
266 var optionalPom = repo.pom(rootGroupAndArtifactId);
267
268 if (optionalPom.isPresent() && optionalPom.get() instanceof Pom pom) {
269 pom.requiredDependencies()
270 .filter(dependencyId -> !dependencyId.scope.equals(Scope.TEST))
271 .forEach(
272 dependencyId -> {
273 add(metaDataId, dependencyId);
274 unresolved.add(dependencyId);
275 });
276 }
277 });
278
279 while (!unresolved.isEmpty()) {
280 var resolveSet = new HashSet<>(unresolved);
281 unresolved.clear();
282 resolveSet.forEach(
283 id -> {
284 if (repo.pom(id) instanceof Optional<Pom> p && p.isPresent()) {
285 p.get()
286 .requiredDependencies()
287 .filter(dependencyId -> !dependencyId.scope.equals(Scope.TEST))
288 .forEach(
289 dependencyId -> {
290 unresolved.add(dependencyId);
291 add(id, dependencyId);
292 });
293 // }else{
294 // throw new IllegalArgumentException("unresolved pom " + id);
295 }
296 });
297 }
298 removeUNSPECIFIED();
299 }
300
301 @Override public List<Bldr.ClassPathEntry> classPathEntries(){
302 return classPath().classPathEntries();
303 }
304
305 Bldr.ClassPath classPath() {
306
307 Bldr.ClassPath jars = Bldr.ClassPath.of();
308 nodes
309 .keySet()
310 .forEach(
311 id -> {
312 // println("looking for pom for "+id);
313 Optional<Pom> optionalPom = repo.pom(id);
314 if (optionalPom.isPresent() && optionalPom.get() instanceof Pom pom) {
315 jars.add(pom.getJar());
316 } else {
317 throw new RuntimeException("No pom for " + id + " needed by " + id);
318 }
319 });
320 return jars;
321 }
322 }
323
324 public Dag dag(String... rootGroupAndArtifactIds) {
325 return dag(Stream.of(rootGroupAndArtifactIds).map(GroupAndArtifactId::of).toList());
326 }
327
328 public Dag dag(GroupAndArtifactId... rootGroupAndArtifactIds) {
329 return dag(List.of(rootGroupAndArtifactIds));
330 }
331
332 public Dag dag(List<GroupAndArtifactId> rootGroupAndArtifactIds) {
333 var dag = new Dag(this, rootGroupAndArtifactIds);
334 return dag;
335 }
336
337
338 public record VersionId(Integer maj, Integer min, Integer point, String classifier)
339 implements Comparable<VersionId> {
340 static Integer integerOrNull(String s) {
341 return (s == null || s.isEmpty()) ? null : Integer.parseInt(s);
342 }
343
344 public static Pattern pattern = Pattern.compile("^(\\d+)(?:\\.(\\d+)(?:\\.(\\d+)(.*))?)?$");
345 static VersionId UNSPECIFIED = new VersionId(null, null, null, null);
346
347 static VersionId of(String version) {
348 Matcher matcher = pattern.matcher(version);
349 if (matcher.matches()) {
350 return new VersionId(
351 integerOrNull(matcher.group(1)),
352 integerOrNull(matcher.group(2)),
353 integerOrNull(matcher.group(3)),
354 matcher.group(4));
355 } else {
356 return UNSPECIFIED;
357 }
358 }
359
360 int cmp(Integer v1, Integer v2) {
361 if (v1 == null && v2 == null) {
362 return 0;
363 }
364 if (v1 == null) {
365 return -v2;
366 } else if (v2 == null) {
367 return v1;
368 } else {
369 return v1 - v2;
370 }
371 }
372
373 @Override
374 public int compareTo(VersionId o) {
375 if (cmp(maj(), o.maj()) == 0) {
376 if (cmp(min(), o.min()) == 0) {
377 if (cmp(point(), o.point()) == 0) {
378 return classifier().compareTo(o.classifier());
379 } else {
380 return cmp(point(), o.point());
381 }
382 } else {
383 return cmp(min(), o.min());
384 }
385 } else {
386 return cmp(maj(), o.maj());
387 }
388 }
389
390 @Override
391 public String toString() {
392 StringBuilder sb = new StringBuilder();
393 if (maj() != null) {
394 sb.append(maj());
395 if (min() != null) {
396 sb.append(".").append(min());
397 if (point() != null) {
398 sb.append(".").append(point());
399 if (classifier() != null) {
400 sb.append(classifier());
401 }
402 }
403 }
404 } else {
405 sb.append("UNSPECIFIED");
406 }
407 return sb.toString();
408 }
409 }
410
411 public record GroupId(String string) {
412 public static GroupId of(String s) {
413 return new GroupId(s);
414 }
415
416 @Override
417 public String toString() {
418 return string;
419 }
420 }
421
422 public record ArtifactId(String string) {
423 static ArtifactId of(String string) {
424 return new ArtifactId(string);
425 }
426
427 @Override
428 public String toString() {
429 return string;
430 }
431 }
432
433 public record MetaDataId(
434 MavenStyleRepository mavenStyleRepository,
435 GroupAndArtifactId groupAndArtifactId,
436 VersionId versionId,
437 Set<String> downloadables,
438 Set<String> tags)
439 implements Id {
440
441 public URL pomURL() {
442 return url("pom");
443 }
444
445 public URL jarURL() {
446 return url("jar");
447 }
448
449 public XMLNode getPom() {
450 if (downloadables.contains(".pom")) {
451 return mavenStyleRepository.queryAndCache(
452 url("pom"), mavenStyleRepository.dir.xmlFile(artifactAndVersion() + ".pom"));
453 } else {
454 throw new IllegalStateException("no pom");
455 }
456 }
457
458 @Override
459 public String toString() {
460 return groupAndArtifactId().toString() + "." + versionId();
461 }
462 }
463
464 public MavenStyleRepository(Bldr.BuildDir dir) {
465 this.dir = dir.create();
466 }
467
468 Bldr.JarFile queryAndCache(URL query, Bldr.JarFile jarFile) {
469 try {
470 if (!jarFile.exists()) {
471 print("Querying and caching " + jarFile.fileName());
472 println(" downloading " + query);
473 curl(query, jarFile.path());
474 } else {
475 // println("Using cached " + jarFile.fileName());
476
477 }
478 } catch (Throwable e) {
479 throw new RuntimeException(e);
480 }
481 return jarFile;
482 }
483
484 XMLNode queryAndCache(URL query, Bldr.XMLFile xmlFile) {
485 XMLNode xmlNode = null;
486 try {
487 if (!xmlFile.exists()) {
488 print("Querying and caching " + xmlFile.fileName());
489 println(" downloading " + query);
490 xmlNode = new XMLNode(query);
491 xmlNode.write(xmlFile.path().toFile());
492 } else {
493 // println("Using cached " + xmlFile.fileName());
494 xmlNode = new XMLNode(xmlFile.path());
495 }
496 } catch (Throwable e) {
497 throw new RuntimeException(e);
498 }
499 return xmlNode;
500 }
501
502 public record MetaData(
503 MavenStyleRepository mavenStyleRepository,
504 GroupAndArtifactId groupAndArtifactId,
505 XMLNode xmlNode) {
506
507 public Stream<MetaDataId> metaDataIds() {
508 return xmlNode
509 .xmlNodes(xmlNode.xpath("/response/result/doc"))
510 .map(
511 xmln ->
512 new MetaDataId(
513 this.mavenStyleRepository,
514 GroupAndArtifactId.of(
515 GroupId.of(xmln.xpathQueryString("str[@name='g']/text()")),
516 ArtifactId.of(xmln.xpathQueryString("str[@name='a']/text()"))),
517 VersionId.of(xmln.xpathQueryString("str[@name='v']/text()")),
518 new HashSet<>(
519 xmln.nodes(xmln.xpath("arr[@name='ec']/str/text()"))
520 .map(Node::getNodeValue)
521 .toList()),
522 new HashSet<>(
523 xmln.nodes(xmln.xpath("arr[@name='tags']/str/text()"))
524 .map(Node::getNodeValue)
525 .toList())));
526 }
527
528 public Stream<MetaDataId> sortedMetaDataIds() {
529 return metaDataIds().sorted(Comparator.comparing(MetaDataId::versionId));
530 }
531
532 public Optional<MetaDataId> latestMetaDataId() {
533 return metaDataIds().max(Comparator.comparing(MetaDataId::versionId));
534 }
535
536 public Optional<MetaDataId> getMetaDataId(VersionId versionId) {
537 return metaDataIds().filter(id -> versionId.compareTo(id.versionId()) == 0).findFirst();
538 }
539 }
540
541 public Optional<MetaData> metaData(String groupId, String artifactId) {
542 return metaData(GroupAndArtifactId.of(groupId, artifactId));
543 }
544
545 public Optional<MetaData> metaData(GroupAndArtifactId groupAndArtifactId) {
546 try {
547 var query = "g:" + groupAndArtifactId.groupId() + " AND a:" + groupAndArtifactId.artifactId();
548 URL rowQueryUrl =
549 new URI(
550 searchBase
551 + "select?q="
552 + URLEncoder.encode(query, StandardCharsets.UTF_8)
553 + "&core=gav&wt=xml&rows=0")
554 .toURL();
555 var rowQueryResponse = new XMLNode(rowQueryUrl);
556 var numFound = rowQueryResponse.xpathQueryString("/response/result/@numFound");
557
558 URL url =
559 new URI(
560 searchBase
561 + "select?q="
562 + URLEncoder.encode(query, StandardCharsets.UTF_8)
563 + "&core=gav&wt=xml&rows="
564 + numFound)
565 .toURL();
566 try {
567 // println(url);
568 var xmlNode =
569 queryAndCache(url, dir.xmlFile(groupAndArtifactId.artifactId() + ".meta.xml"));
570 // var numFound2 = xmlNode.xpathQueryString("/response/result/@numFound");
571 // var start = xmlNode.xpathQueryString("/response/result/@start");
572 // var rows =
573 // xmlNode.xpathQueryString("/response/lst[@name='responseHeader']/lst[@name='params']/str[@name='rows']/text()");
574 // println("numFound = "+numFound+" rows ="+rows+ " start ="+start);
575 if (numFound.isEmpty() || numFound.equals("0")) {
576 return Optional.empty();
577 } else {
578 return Optional.of(new MetaData(this, groupAndArtifactId, xmlNode));
579 }
580 } catch (Throwable e) {
581 throw new RuntimeException(e);
582 }
583 } catch (Throwable e) {
584 throw new RuntimeException(e);
585 }
586 }
587
588 public static void main(String[] args) {
589
590 var hatDir = assertExists(Bldr.Dir.of("/Users/grfrost/github/babylon-grfrost-fork/hat"));
591
592 var repo = new MavenStyleRepository(hatDir.buildDir("repo"));
593 var testngMeta = repo.metaData("org.testng", "testng");
594 if (testngMeta.isPresent()) {
595 testngMeta
596 .get()
597 .sortedMetaDataIds()
598 .forEach(
599 metaDataId -> {
600 var pom = metaDataId.getPom();
601 // println(pom.toString());
602 });
603 }
604 var junitMeta = repo.metaData("org.junit.platform", "junit-platform-console-standalone");
605 if (junitMeta.isPresent()) {
606 var latest = junitMeta.get().latestMetaDataId();
607 if (latest.isPresent()) {
608 var id = latest.get();
609 var pom = id.getPom();
610 // println(pom.toString());
611 }
612 }
613 }
614 }