1 /*
  2  * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2023, Red Hat Inc.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  */
 24 
 25 /*
 26  * @test id=ENABLED
 27  * @bug 8303215 8312182
 28  * @summary On THP=always systems, we prevent THPs from forming within thread stacks
 29  * @library /test/lib
 30  * @requires os.family == "linux"
 31  * @requires vm.debug
 32  * @requires os.arch=="amd64" | os.arch=="x86_64" | os.arch=="aarch64"
 33  * @modules java.base/jdk.internal.misc
 34  *          java.management
 35  * @run driver THPsInThreadStackPreventionTest PATCH-ENABLED
 36  */
 37 
 38 /*
 39  * @test id=DISABLED
 40  * @bug 8303215 8312182
 41  * @summary On THP=always systems, we prevent THPs from forming within thread stacks (negative test)
 42  * @library /test/lib
 43  * @requires os.family == "linux"
 44  * @requires vm.debug
 45  * @requires os.arch=="amd64" | os.arch=="x86_64" | os.arch=="aarch64"
 46  * @modules java.base/jdk.internal.misc
 47  *          java.management
 48  * @run main/manual THPsInThreadStackPreventionTest  PATCH-DISABLED
 49  */
 50 import jdk.test.lib.process.OutputAnalyzer;
 51 import jdk.test.lib.process.ProcessTools;
 52 import jtreg.SkippedException;
 53 
 54 import java.io.BufferedReader;
 55 import java.io.FileReader;
 56 import java.io.IOException;
 57 import java.util.ArrayList;
 58 import java.util.Arrays;
 59 import java.util.Objects;
 60 import java.util.concurrent.BrokenBarrierException;
 61 import java.util.concurrent.CyclicBarrier;
 62 
 63 public class THPsInThreadStackPreventionTest {
 64 
 65     // We test the mitigation for "huge rss for THP=always" introduced with JDK-8312182 and JDK-8302015:
 66     //
 67     // We start a program that spawns a ton of threads with a stack size close to THP page size. The threads
 68     // are idle and should not build up a lot of stack. The threads are started with an artificial delay
 69     // between thread start and stack guardpage creation, which exacerbates the RSS bloat (for explanation
 70     // please see 8312182).
 71     //
 72     // We then observe RSS of that program. We expect it to stay below a reasonable maximum. The unpatched
 73     // version should show an RSS of ~2 GB (paying for the fully paged in thread stacks). The fixed variant should
 74     // cost only ~200-400 MB.
 75 
 76     static final int numThreads = 1000;
 77     static final long threadStackSizeMB = 2; // must be 2M
 78     static final long heapSizeMB = 64;
 79     static final long basicRSSOverheadMB = heapSizeMB + 150;
 80     // A successful completion of this test would show not more than X KB per thread stack.
 81     static final long acceptableRSSPerThreadStack = 128 * 1024;
 82     static final long acceptableRSSForAllThreadStacks = numThreads * acceptableRSSPerThreadStack;
 83     static final long acceptableRSSLimitMB = (acceptableRSSForAllThreadStacks / (1024 * 1024)) + basicRSSOverheadMB;
 84 
 85     private static class TestMain {
 86 
 87         static class Sleeper extends Thread {
 88             CyclicBarrier barrier;
 89             public Sleeper(CyclicBarrier barrier) {
 90                 this.barrier = barrier;
 91             }
 92             @Override
 93             public void run() {
 94                 try {
 95                     barrier.await(); // wait for all siblings
 96                     barrier.await(); // wait main thread to print status
 97                 } catch (InterruptedException | BrokenBarrierException e) {
 98                     e.printStackTrace();
 99                 }
100             }
101         }
102 
103         public static void main(String[] args) throws BrokenBarrierException, InterruptedException {
104 
105             // Fire up 1000 threads with 2M stack size each.
106             Sleeper[] threads = new Sleeper[numThreads];
107             CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
108 
109             for (int i = 0; i < numThreads; i++) {
110                 threads[i] = new Sleeper(barrier);
111                 threads[i].start();
112             }
113 
114             // Wait for all threads to come up
115             barrier.await();
116 
117             // print status
118             String file = "/proc/self/status";
119             try (FileReader fr = new FileReader(file);
120                  BufferedReader reader = new BufferedReader(fr)) {
121                 String line;
122                 while ((line = reader.readLine()) != null) {
123                     System.out.println(line);
124                 }
125             } catch (IOException | NumberFormatException e) { /* ignored */ }
126 
127             // Signal threads to stop
128             barrier.await();
129 
130         }
131     }
132 
133     static class ProcSelfStatus {
134 
135         public long rssMB;
136         public long swapMB;
137         public int numLifeThreads;
138 
139         // Parse output from /proc/self/status
140         public static ProcSelfStatus parse(OutputAnalyzer o) {
141             ProcSelfStatus status = new ProcSelfStatus();
142             String s = o.firstMatch("Threads:\\s*(\\d+)", 1);
143             Objects.requireNonNull(s);
144             status.numLifeThreads = Integer.parseInt(s);
145             s = o.firstMatch("VmRSS:\\s*(\\d+) kB", 1);
146             Objects.requireNonNull(s);
147             status.rssMB = Long.parseLong(s) / 1024;
148             s = o.firstMatch("VmSwap:\\s*(\\d+) kB", 1);
149             Objects.requireNonNull(s);
150             status.swapMB = Long.parseLong(s) / 1024;
151             return status;
152         }
153     }
154 
155     public static void main(String[] args) throws Exception {
156 
157         HugePageConfiguration config = HugePageConfiguration.readFromOS();
158         // This issue is bound to THP=always
159         if (config.getThpMode() != HugePageConfiguration.THPMode.always) {
160             throw new SkippedException("Test only makes sense in THP \"always\" mode");
161         }
162 
163         String[] defaultArgs = {
164             "-Xlog:pagesize",
165             "-Xmx" + heapSizeMB + "m", "-Xms" + heapSizeMB + "m", "-XX:+AlwaysPreTouch", // stabilize RSS
166             "-Xss" + threadStackSizeMB + "m",
167             "-XX:-CreateCoredumpOnCrash",
168             // Limits the number of JVM-internal threads, which depends on the available cores of the
169             // machine. RSS+Swap could exceed acceptableRSSLimitMB when JVM creates many internal threads.
170             "-XX:ActiveProcessorCount=2",
171             // This will delay the child threads before they create guard pages, thereby greatly increasing the
172             // chance of large VMA formation + hugepage coalescation; see JDK-8312182
173             "-XX:+DelayThreadStartALot"
174         };
175         ArrayList<String> finalargs = new ArrayList<>(Arrays.asList(defaultArgs));
176 
177         switch (args[0]) {
178             case "PATCH-ENABLED": {
179                 finalargs.add(TestMain.class.getName());
180                 ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(finalargs);
181 
182                 OutputAnalyzer output = new OutputAnalyzer(pb.start());
183                 output.shouldHaveExitValue(0);
184 
185                 // this line indicates the mitigation is active:
186                 output.shouldContain("[pagesize] JVM will attempt to prevent THPs in thread stacks.");
187 
188                 ProcSelfStatus status = ProcSelfStatus.parse(output);
189                 if (status.numLifeThreads < numThreads) {
190                     throw new RuntimeException("Number of live threads lower than expected: " + status.numLifeThreads + ", expected " + numThreads);
191                 } else {
192                     System.out.println("Found " + status.numLifeThreads + " to be alive. Ok.");
193                 }
194 
195                 long rssPlusSwapMB = status.swapMB + status.rssMB;
196 
197                 if (rssPlusSwapMB > acceptableRSSLimitMB) {
198                     throw new RuntimeException("RSS+Swap larger than expected: " + rssPlusSwapMB + "m, expected at most " + acceptableRSSLimitMB + "m");
199                 } else {
200                     if (rssPlusSwapMB < heapSizeMB) { // we pretouch the java heap, so we expect to see at least that:
201                         throw new RuntimeException("RSS+Swap suspiciously low: " + rssPlusSwapMB + "m, expected at least " + heapSizeMB + "m");
202                     }
203                     System.out.println("Okay: RSS+Swap=" + rssPlusSwapMB + ", within acceptable limit of " + acceptableRSSLimitMB);
204                 }
205             }
206             break;
207 
208             case "PATCH-DISABLED": {
209 
210                 // Only execute manually! this will allocate ~2gb of memory!
211 
212                 // explicitly disable the no-THP-workaround:
213                 finalargs.add("-XX:+UnlockDiagnosticVMOptions");
214                 finalargs.add("-XX:-THPStackMitigation");
215 
216                 finalargs.add(TestMain.class.getName());
217                 ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(finalargs);
218                 OutputAnalyzer output = new OutputAnalyzer(pb.start());
219 
220                 output.shouldHaveExitValue(0);
221 
222                 // We deliberately switched off mitigation, VM should tell us:
223                 output.shouldContain("[pagesize] JVM will *not* prevent THPs in thread stacks. This may cause high RSS.");
224 
225                 // Parse output from self/status
226                 ProcSelfStatus status = ProcSelfStatus.parse(output);
227                 if (status.numLifeThreads < numThreads) {
228                     throw new RuntimeException("Number of live threads lower than expected (" + status.numLifeThreads + ", expected " + numThreads +")");
229                 } else {
230                     System.out.println("Found " + status.numLifeThreads + " to be alive. Ok.");
231                 }
232 
233                 long rssPlusSwapMB = status.swapMB + status.rssMB;
234 
235                 if (rssPlusSwapMB < acceptableRSSLimitMB) {
236                     throw new RuntimeException("RSS+Swap lower than expected: " + rssPlusSwapMB + "m, expected more than " + acceptableRSSLimitMB + "m");
237                 }
238                 break;
239             }
240 
241             default: throw new RuntimeException("Bad argument: " + args[0]);
242         }
243     }
244 }