1 /*
  2  * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2023, Red Hat Inc.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  */
 24 
 25 /*
 26  * @test id=ENABLED
 27  * @bug 8303215 8312182
 28  * @summary On THP=always systems, we prevent THPs from forming within thread stacks
 29  * @library /test/lib
 30  * @requires vm.flagless
 31  * @requires os.family == "linux"
 32  * @requires vm.debug
 33  * @requires os.arch=="amd64" | os.arch=="x86_64" | os.arch=="aarch64"
 34  * @modules java.base/jdk.internal.misc
 35  *          java.management
 36  * @run driver THPsInThreadStackPreventionTest PATCH-ENABLED
 37  */
 38 
 39 /*
 40  * @test id=DISABLED
 41  * @bug 8303215 8312182
 42  * @summary On THP=always systems, we prevent THPs from forming within thread stacks (negative test)
 43  * @library /test/lib
 44  * @requires vm.flagless
 45  * @requires os.family == "linux"
 46  * @requires vm.debug
 47  * @requires os.arch=="amd64" | os.arch=="x86_64" | os.arch=="aarch64"
 48  * @modules java.base/jdk.internal.misc
 49  *          java.management
 50  * @run main/manual THPsInThreadStackPreventionTest  PATCH-DISABLED
 51  */
 52 import jdk.test.lib.process.OutputAnalyzer;
 53 import jdk.test.lib.process.ProcessTools;
 54 import jtreg.SkippedException;
 55 
 56 import java.io.BufferedReader;
 57 import java.io.FileReader;
 58 import java.io.IOException;
 59 import java.util.ArrayList;
 60 import java.util.Arrays;
 61 import java.util.Objects;
 62 import java.util.concurrent.BrokenBarrierException;
 63 import java.util.concurrent.CyclicBarrier;
 64 
 65 public class THPsInThreadStackPreventionTest {
 66 
 67     // We test the mitigation for "huge rss for THP=always" introduced with JDK-8312182 and JDK-8302015:
 68     //
 69     // We start a program that spawns a ton of threads with a stack size close to THP page size. The threads
 70     // are idle and should not build up a lot of stack. The threads are started with an artificial delay
 71     // between thread start and stack guardpage creation, which exacerbates the RSS bloat (for explanation
 72     // please see 8312182).
 73     //
 74     // We then observe RSS of that program. We expect it to stay below a reasonable maximum. The unpatched
 75     // version should show an RSS of ~2 GB (paying for the fully paged in thread stacks). The fixed variant should
 76     // cost only ~200-400 MB.
 77 
 78     static final int numThreads = 1000;
 79     static final long threadStackSizeMB = 2; // must be 2M
 80     static final long heapSizeMB = 64;
 81     static final long basicRSSOverheadMB = heapSizeMB + 150;
 82     // A successful completion of this test would show not more than X KB per thread stack.
 83     static final long acceptableRSSPerThreadStack = 128 * 1024;
 84     static final long acceptableRSSForAllThreadStacks = numThreads * acceptableRSSPerThreadStack;
 85     static final long acceptableRSSLimitMB = (acceptableRSSForAllThreadStacks / (1024 * 1024)) + basicRSSOverheadMB;
 86 
 87     private static class TestMain {
 88 
 89         static class Sleeper extends Thread {
 90             CyclicBarrier barrier;
 91             public Sleeper(CyclicBarrier barrier) {
 92                 this.barrier = barrier;
 93             }
 94             @Override
 95             public void run() {
 96                 try {
 97                     barrier.await(); // wait for all siblings
 98                     barrier.await(); // wait main thread to print status
 99                 } catch (InterruptedException | BrokenBarrierException e) {
100                     e.printStackTrace();
101                 }
102             }
103         }
104 
105         public static void main(String[] args) throws BrokenBarrierException, InterruptedException {
106 
107             // Fire up 1000 threads with 2M stack size each.
108             Sleeper[] threads = new Sleeper[numThreads];
109             CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
110 
111             for (int i = 0; i < numThreads; i++) {
112                 threads[i] = new Sleeper(barrier);
113                 threads[i].start();
114             }
115 
116             // Wait for all threads to come up
117             barrier.await();
118 
119             // print status
120             String file = "/proc/self/status";
121             try (FileReader fr = new FileReader(file);
122                  BufferedReader reader = new BufferedReader(fr)) {
123                 String line;
124                 while ((line = reader.readLine()) != null) {
125                     System.out.println(line);
126                 }
127             } catch (IOException | NumberFormatException e) { /* ignored */ }
128 
129             // Signal threads to stop
130             barrier.await();
131 
132         }
133     }
134 
135     static class ProcSelfStatus {
136 
137         public long rssMB;
138         public long swapMB;
139         public int numLifeThreads;
140 
141         // Parse output from /proc/self/status
142         public static ProcSelfStatus parse(OutputAnalyzer o) {
143             ProcSelfStatus status = new ProcSelfStatus();
144             String s = o.firstMatch("Threads:\\s*(\\d+)", 1);
145             Objects.requireNonNull(s);
146             status.numLifeThreads = Integer.parseInt(s);
147             s = o.firstMatch("VmRSS:\\s*(\\d+) kB", 1);
148             Objects.requireNonNull(s);
149             status.rssMB = Long.parseLong(s) / 1024;
150             s = o.firstMatch("VmSwap:\\s*(\\d+) kB", 1);
151             Objects.requireNonNull(s);
152             status.swapMB = Long.parseLong(s) / 1024;
153             return status;
154         }
155     }
156 
157     public static void main(String[] args) throws Exception {
158 
159         HugePageConfiguration config = HugePageConfiguration.readFromOS();
160         // This issue is bound to THP=always
161         if (config.getThpMode() != HugePageConfiguration.THPMode.always) {
162             throw new SkippedException("Test only makes sense in THP \"always\" mode");
163         }
164 
165         String[] defaultArgs = {
166             "-Xlog:pagesize",
167             "-Xmx" + heapSizeMB + "m", "-Xms" + heapSizeMB + "m", "-XX:+AlwaysPreTouch", // stabilize RSS
168             "-Xss" + threadStackSizeMB + "m",
169             "-XX:-CreateCoredumpOnCrash",
170             // Limits the number of JVM-internal threads, which depends on the available cores of the
171             // machine. RSS+Swap could exceed acceptableRSSLimitMB when JVM creates many internal threads.
172             "-XX:ActiveProcessorCount=2",
173             // This will delay the child threads before they create guard pages, thereby greatly increasing the
174             // chance of large VMA formation + hugepage coalescation; see JDK-8312182
175             "-XX:+DelayThreadStartALot"
176         };
177         ArrayList<String> finalargs = new ArrayList<>(Arrays.asList(defaultArgs));
178 
179         switch (args[0]) {
180             case "PATCH-ENABLED": {
181                 finalargs.add(TestMain.class.getName());
182                 ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(finalargs);
183 
184                 OutputAnalyzer output = new OutputAnalyzer(pb.start());
185                 output.shouldHaveExitValue(0);
186 
187                 // this line indicates the mitigation is active:
188                 output.shouldContain("[pagesize] JVM will attempt to prevent THPs in thread stacks.");
189 
190                 ProcSelfStatus status = ProcSelfStatus.parse(output);
191                 if (status.numLifeThreads < numThreads) {
192                     throw new RuntimeException("Number of live threads lower than expected: " + status.numLifeThreads + ", expected " + numThreads);
193                 } else {
194                     System.out.println("Found " + status.numLifeThreads + " to be alive. Ok.");
195                 }
196 
197                 long rssPlusSwapMB = status.swapMB + status.rssMB;
198 
199                 if (rssPlusSwapMB > acceptableRSSLimitMB) {
200                     throw new RuntimeException("RSS+Swap larger than expected: " + rssPlusSwapMB + "m, expected at most " + acceptableRSSLimitMB + "m");
201                 } else {
202                     if (rssPlusSwapMB < heapSizeMB) { // we pretouch the java heap, so we expect to see at least that:
203                         throw new RuntimeException("RSS+Swap suspiciously low: " + rssPlusSwapMB + "m, expected at least " + heapSizeMB + "m");
204                     }
205                     System.out.println("Okay: RSS+Swap=" + rssPlusSwapMB + ", within acceptable limit of " + acceptableRSSLimitMB);
206                 }
207             }
208             break;
209 
210             case "PATCH-DISABLED": {
211 
212                 // Only execute manually! this will allocate ~2gb of memory!
213 
214                 // explicitly disable the no-THP-workaround:
215                 finalargs.add("-XX:+UnlockDiagnosticVMOptions");
216                 finalargs.add("-XX:-THPStackMitigation");
217 
218                 finalargs.add(TestMain.class.getName());
219                 ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(finalargs);
220                 OutputAnalyzer output = new OutputAnalyzer(pb.start());
221 
222                 output.shouldHaveExitValue(0);
223 
224                 // We deliberately switched off mitigation, VM should tell us:
225                 output.shouldContain("[pagesize] JVM will *not* prevent THPs in thread stacks. This may cause high RSS.");
226 
227                 // Parse output from self/status
228                 ProcSelfStatus status = ProcSelfStatus.parse(output);
229                 if (status.numLifeThreads < numThreads) {
230                     throw new RuntimeException("Number of live threads lower than expected (" + status.numLifeThreads + ", expected " + numThreads +")");
231                 } else {
232                     System.out.println("Found " + status.numLifeThreads + " to be alive. Ok.");
233                 }
234 
235                 long rssPlusSwapMB = status.swapMB + status.rssMB;
236 
237                 if (rssPlusSwapMB < acceptableRSSLimitMB) {
238                     throw new RuntimeException("RSS+Swap lower than expected: " + rssPlusSwapMB + "m, expected more than " + acceptableRSSLimitMB + "m");
239                 }
240                 break;
241             }
242 
243             default: throw new RuntimeException("Bad argument: " + args[0]);
244         }
245     }
246 }