diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java b/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java index 61fe010bf16a..3d6b6d6f51cb 100644 --- a/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/JavaVersionUtils.java @@ -24,44 +24,29 @@ private JavaVersionUtils() { } /** - * Returns the detected Java major version (e.g., 8, 11, 17). + * Returns JVM --add-opens flags for Tez/MR child processes. */ - public static int getJavaMajorVersion() { - return Math.max(8, Integer.parseInt( - System.getProperty("java.specification.version").split("\\.")[0])); - } - - /** - * Returns the full --add-opens string if Java 9, or an empty string if Java 8 or below. - */ - public static String getAddOpensFlagsIfNeeded() { - //TODO: Please remove this once the codebase has been migrated to JDK 9 or a higher version. - int javaVersion = getJavaMajorVersion(); - if (javaVersion >= 9) { - return " --add-opens=java.base/java.net=ALL-UNNAMED" + - " --add-opens=java.base/java.util=ALL-UNNAMED" + - " --add-opens=java.base/java.util.concurrent=ALL-UNNAMED" + - " --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" + - " --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" + - " --add-opens=java.base/java.lang=ALL-UNNAMED" + - " --add-opens=java.base/java.io=ALL-UNNAMED" + - " --add-opens=java.base/java.lang=ALL-UNNAMED" + - " --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + - " --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" + - " --add-opens=java.base/java.math=ALL-UNNAMED" + - " --add-opens=java.base/java.nio=ALL-UNNAMED" + - " --add-opens=java.base/java.text=ALL-UNNAMED" + - " --add-opens=java.base/java.time=ALL-UNNAMED" + - " --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" + - " --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" + - " --add-opens=java.sql/java.sql=ALL-UNNAMED" + - " --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" + - " --add-opens=java.base/sun.nio.cs=ALL-UNNAMED" + - " --add-opens=java.base/java.util.regex=ALL-UNNAMED" + - " --add-opens=java.base/java.security=ALL-UNNAMED" + - " --add-opens=java.base/sun.security.provider=ALL-UNNAMED"; - } else { - return ""; // Java 8 or lower does not need add-opens - } + public static String getAddOpensFlags() { + return " --add-opens=java.base/java.net=ALL-UNNAMED" + + " --add-opens=java.base/java.util=ALL-UNNAMED" + + " --add-opens=java.base/java.util.concurrent=ALL-UNNAMED" + + " --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED" + + " --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" + + " --add-opens=java.base/java.lang=ALL-UNNAMED" + + " --add-opens=java.base/java.io=ALL-UNNAMED" + + " --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + + " --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" + + " --add-opens=java.base/java.math=ALL-UNNAMED" + + " --add-opens=java.base/java.nio=ALL-UNNAMED" + + " --add-opens=java.base/java.text=ALL-UNNAMED" + + " --add-opens=java.base/java.time=ALL-UNNAMED" + + " --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" + + " --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" + + " --add-opens=java.sql/java.sql=ALL-UNNAMED" + + " --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" + + " --add-opens=java.base/sun.nio.cs=ALL-UNNAMED" + + " --add-opens=java.base/java.util.regex=ALL-UNNAMED" + + " --add-opens=java.base/java.security=ALL-UNNAMED" + + " --add-opens=java.base/sun.security.provider=ALL-UNNAMED"; } } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java index 7253e31b0347..e6a5aa078d34 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java @@ -124,7 +124,7 @@ public int run(String[] args) throws IOException, InterruptedException, ClassNot conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB); } String amJavaOpts = StringUtils.defaultString(appConf.controllerAMChildOpts()); - amJavaOpts += JavaVersionUtils.getAddOpensFlagsIfNeeded(); + amJavaOpts += JavaVersionUtils.getAddOpensFlags(); if (!amJavaOpts.isEmpty()) { conf.set(AppConfig.HADOOP_MR_AM_JAVA_OPTS, amJavaOpts); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index e96ca2c06979..719b1dbcc8c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -374,7 +374,7 @@ public int execute() { } } - addOpensFlagsIfNeeded(job); + addOpensFlags(job); jc = new JobClient(job); // make this client wait if job tracker is not behaving well. Throttle.checkJobTracker(job, LOG); @@ -493,13 +493,11 @@ public int execute() { return (returnVal); } - private static void addOpensFlagsIfNeeded(JobConf job) { - String addOpens = JavaVersionUtils.getAddOpensFlagsIfNeeded(); - if(!addOpens.isEmpty()) { - addJavaOpts(job, "mapreduce.map.java.opts", addOpens); - addJavaOpts(job, "mapreduce.reduce.java.opts", addOpens); - addJavaOpts(job, "yarn.app.mapreduce.am.command-opts", addOpens); - } + private static void addOpensFlags(JobConf job) { + String addOpens = JavaVersionUtils.getAddOpensFlags(); + addJavaOpts(job, "mapreduce.map.java.opts", addOpens); + addJavaOpts(job, "mapreduce.reduce.java.opts", addOpens); + addJavaOpts(job, "yarn.app.mapreduce.am.command-opts", addOpens); } private static void addJavaOpts(JobConf job, String conf, String extraOpts) { String javaOpts = StringUtils.defaultString(job.get(conf)); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index a36364956c79..7921295c4c4f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -728,7 +728,7 @@ private static String getContainerJavaOpts(Configuration conf) { } finalOpts = logLevel + " " + MRHelpers.getJavaOptsForMRMapper(conf); } - finalOpts += JavaVersionUtils.getAddOpensFlagsIfNeeded(); + finalOpts += JavaVersionUtils.getAddOpensFlags(); LOG.debug("Tez container final opts: {}", finalOpts); return finalOpts; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index b50c15cf9b63..70b9b99144eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -582,7 +582,7 @@ private void setupTezParamsBasedOnMR(TezConfiguration conf) { } String mrAmJavaOpts = conf.get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, MRHelpers.getJavaOptsForMRAM(conf)); - conf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, mrAmJavaOpts + JavaVersionUtils.getAddOpensFlagsIfNeeded()); + conf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, mrAmJavaOpts + JavaVersionUtils.getAddOpensFlags()); String queueName = conf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME); conf.setIfUnset(TezConfiguration.TEZ_QUEUE_NAME, queueName);