From ed6a973dd59e8183309bd14c0341d3fbd8ef2c01 Mon Sep 17 00:00:00 2001 From: Daniel Tenedorio Date: Mon, 2 Mar 2026 13:47:29 -0800 Subject: [PATCH] commit --- .../spark/internal/config/ConfigBuilder.scala | 44 ++-- .../spark/internal/config/ConfigEntry.scala | 50 ++++- .../spark/RecordConfigAccessSuite.scala | 200 ++++++++++++++++++ 3 files changed, 265 insertions(+), 29 deletions(-) create mode 100644 core/src/test/scala/org/apache/spark/RecordConfigAccessSuite.scala diff --git a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala index c96324608ba50..538ca22c7de8b 100644 --- a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala +++ b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala @@ -150,17 +150,18 @@ private object ConfigHelpers { private[spark] class TypedConfigBuilder[T]( val parent: ConfigBuilder, val converter: String => T, - val stringConverter: T => String) { + val stringConverter: T => String, + val configEntryType: ConfigEntryType) { import ConfigHelpers._ - def this(parent: ConfigBuilder, converter: String => T) = { - this(parent, converter, { v: T => v.toString }) + def this(parent: ConfigBuilder, converter: String => T, configEntryType: ConfigEntryType) = { + this(parent, converter, { v: T => v.toString }, configEntryType) } /** Apply a transformation to the user-provided values of the config entry. */ def transform(fn: T => T): TypedConfigBuilder[T] = { - new TypedConfigBuilder(parent, s => fn(converter(s)), stringConverter) + new TypedConfigBuilder(parent, s => fn(converter(s)), stringConverter, configEntryType) } /** Checks if the user-provided value for the config matches the validator. */ @@ -204,14 +205,15 @@ private[spark] class TypedConfigBuilder[T]( /** Turns the config entry into a sequence of values of the underlying type. */ def toSequence: TypedConfigBuilder[Seq[T]] = { - new TypedConfigBuilder(parent, stringToSeq(_, converter), seqToString(_, stringConverter)) + new TypedConfigBuilder(parent, stringToSeq(_, converter), seqToString(_, stringConverter), + configEntryType) } /** Creates a [[ConfigEntry]] that does not have a default value. */ def createOptional: OptionalConfigEntry[T] = { val entry = new OptionalConfigEntry[T](parent.key, parent._prependedKey, parent._prependSeparator, parent._alternatives, converter, stringConverter, parent._doc, - parent._public, parent._version) + parent._public, parent._version, configEntryType) parent._onCreate.foreach(_(entry)) entry } @@ -227,7 +229,7 @@ private[spark] class TypedConfigBuilder[T]( val transformedDefault = converter(stringConverter(default)) val entry = new ConfigEntryWithDefault[T](parent.key, parent._prependedKey, parent._prependSeparator, parent._alternatives, transformedDefault, converter, - stringConverter, parent._doc, parent._public, parent._version) + stringConverter, parent._doc, parent._public, parent._version, configEntryType) parent._onCreate.foreach(_ (entry)) entry } @@ -237,7 +239,7 @@ private[spark] class TypedConfigBuilder[T]( def createWithDefaultFunction(defaultFunc: () => T): ConfigEntry[T] = { val entry = new ConfigEntryWithDefaultFunction[T](parent.key, parent._prependedKey, parent._prependSeparator, parent._alternatives, defaultFunc, converter, stringConverter, - parent._doc, parent._public, parent._version) + parent._doc, parent._public, parent._version, configEntryType) parent._onCreate.foreach(_ (entry)) entry } @@ -249,7 +251,7 @@ private[spark] class TypedConfigBuilder[T]( def createWithDefaultString(default: String): ConfigEntry[T] = { val entry = new ConfigEntryWithDefaultString[T](parent.key, parent._prependedKey, parent._prependSeparator, parent._alternatives, default, converter, stringConverter, - parent._doc, parent._public, parent._version) + parent._doc, parent._public, parent._version, configEntryType) parent._onCreate.foreach(_(entry)) entry } @@ -310,46 +312,49 @@ private[spark] case class ConfigBuilder(key: String) { def intConf: TypedConfigBuilder[Int] = { checkPrependConfig - new TypedConfigBuilder(this, toNumber(_, _.toInt, key, "int")) + new TypedConfigBuilder(this, toNumber(_, _.toInt, key, "int"), ConfigEntryType.IntEntry) } def longConf: TypedConfigBuilder[Long] = { checkPrependConfig - new TypedConfigBuilder(this, toNumber(_, _.toLong, key, "long")) + new TypedConfigBuilder(this, toNumber(_, _.toLong, key, "long"), ConfigEntryType.LongEntry) } def doubleConf: TypedConfigBuilder[Double] = { checkPrependConfig - new TypedConfigBuilder(this, toNumber(_, _.toDouble, key, "double")) + new TypedConfigBuilder(this, toNumber(_, _.toDouble, key, "double"), + ConfigEntryType.DoubleEntry) } def booleanConf: TypedConfigBuilder[Boolean] = { checkPrependConfig - new TypedConfigBuilder(this, toBoolean(_, key)) + new TypedConfigBuilder(this, toBoolean(_, key), ConfigEntryType.BooleanEntry) } def stringConf: TypedConfigBuilder[String] = { - new TypedConfigBuilder(this, v => v) + new TypedConfigBuilder(this, v => v, ConfigEntryType.StringEntry) } def enumConf(e: Enumeration): TypedConfigBuilder[e.Value] = { checkPrependConfig - new TypedConfigBuilder(this, toEnum(_, e, key)) + new TypedConfigBuilder(this, toEnum(_, e, key), ConfigEntryType.EnumEntry) } def enumConf[E <: Enum[E]](e: Class[E]): TypedConfigBuilder[E] = { checkPrependConfig - new TypedConfigBuilder(this, toEnum(_, e, key)) + new TypedConfigBuilder(this, toEnum(_, e, key), ConfigEntryType.EnumEntry) } def timeConf(unit: TimeUnit): TypedConfigBuilder[Long] = { checkPrependConfig - new TypedConfigBuilder(this, timeFromString(_, unit, key), timeToString(_, unit)) + new TypedConfigBuilder(this, timeFromString(_, unit, key), timeToString(_, unit), + ConfigEntryType.TimeEntry) } def bytesConf(unit: ByteUnit): TypedConfigBuilder[Long] = { checkPrependConfig - new TypedConfigBuilder(this, byteFromString(_, unit, key), byteToString(_, unit, key)) + new TypedConfigBuilder(this, byteFromString(_, unit, key), byteToString(_, unit, key), + ConfigEntryType.BytesEntry) } def fallbackConf[T](fallback: ConfigEntry[T]): ConfigEntry[T] = { @@ -361,7 +366,8 @@ private[spark] case class ConfigBuilder(key: String) { def regexConf: TypedConfigBuilder[Regex] = { checkPrependConfig - new TypedConfigBuilder(this, regexFromString(_, this.key), _.toString) + new TypedConfigBuilder(this, regexFromString(_, this.key), _.toString, + ConfigEntryType.RegexEntry) } private def checkPrependConfig = { diff --git a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala index 17d3329e6b494..347c59e958f9f 100644 --- a/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala +++ b/common/utils/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala @@ -17,6 +17,26 @@ package org.apache.spark.internal.config +/** + * Identifies the declared type of a [[ConfigEntry]] at construction time, + * avoiding runtime type probing or exception handling when the entry type + * needs to be inspected later (e.g. for optimized config access paths). + */ +private[spark] sealed trait ConfigEntryType extends Serializable + +private[spark] object ConfigEntryType { + case object BooleanEntry extends ConfigEntryType + case object IntEntry extends ConfigEntryType + case object LongEntry extends ConfigEntryType + case object DoubleEntry extends ConfigEntryType + case object StringEntry extends ConfigEntryType + case object EnumEntry extends ConfigEntryType + case object TimeEntry extends ConfigEntryType + case object BytesEntry extends ConfigEntryType + case object RegexEntry extends ConfigEntryType + case object OtherEntry extends ConfigEntryType +} + // ==================================================================================== // The guideline for naming configurations // ==================================================================================== @@ -80,7 +100,8 @@ private[spark] abstract class ConfigEntry[T] ( val stringConverter: T => String, val doc: String, val isPublic: Boolean, - val version: String) { + val version: String, + val configEntryType: ConfigEntryType) { import ConfigEntry._ @@ -120,7 +141,8 @@ private class ConfigEntryWithDefault[T] ( stringConverter: T => String, doc: String, isPublic: Boolean, - version: String) + version: String, + configEntryType: ConfigEntryType) extends ConfigEntry( key, prependedKey, @@ -130,7 +152,8 @@ private class ConfigEntryWithDefault[T] ( stringConverter, doc, isPublic, - version + version, + configEntryType ) { override def defaultValue: Option[T] = Some(_defaultValue) @@ -152,7 +175,8 @@ private class ConfigEntryWithDefaultFunction[T] ( stringConverter: T => String, doc: String, isPublic: Boolean, - version: String) + version: String, + configEntryType: ConfigEntryType) extends ConfigEntry( key, prependedKey, @@ -162,7 +186,8 @@ private class ConfigEntryWithDefaultFunction[T] ( stringConverter, doc, isPublic, - version + version, + configEntryType ) { override def defaultValue: Option[T] = Some(_defaultFunction()) @@ -184,7 +209,8 @@ private class ConfigEntryWithDefaultString[T] ( stringConverter: T => String, doc: String, isPublic: Boolean, - version: String) + version: String, + configEntryType: ConfigEntryType) extends ConfigEntry( key, prependedKey, @@ -194,7 +220,8 @@ private class ConfigEntryWithDefaultString[T] ( stringConverter, doc, isPublic, - version + version, + configEntryType ) { override def defaultValue: Option[T] = Some(valueConverter(_defaultValue)) @@ -220,7 +247,8 @@ private[spark] class OptionalConfigEntry[T]( val rawStringConverter: T => String, doc: String, isPublic: Boolean, - version: String) + version: String, + configEntryType: ConfigEntryType) extends ConfigEntry[Option[T]]( key, prependedKey, @@ -230,7 +258,8 @@ private[spark] class OptionalConfigEntry[T]( v => v.map(rawStringConverter).orNull, doc, isPublic, - version + version, + configEntryType ) { override def defaultValueString: String = ConfigEntry.UNDEFINED @@ -261,7 +290,8 @@ private[spark] class FallbackConfigEntry[T] ( fallback.stringConverter, doc, isPublic, - version + version, + fallback.configEntryType ) { override def defaultValueString: String = s"" diff --git a/core/src/test/scala/org/apache/spark/RecordConfigAccessSuite.scala b/core/src/test/scala/org/apache/spark/RecordConfigAccessSuite.scala new file mode 100644 index 0000000000000..93e2df11d7e8b --- /dev/null +++ b/core/src/test/scala/org/apache/spark/RecordConfigAccessSuite.scala @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark + +import java.util.concurrent.TimeUnit + +import org.apache.spark.internal.config._ +import org.apache.spark.internal.config.ConfigEntryType._ +import org.apache.spark.network.util.ByteUnit + +/** + * Tests for the ConfigEntry.configEntryType enum and its assignment + * through ConfigBuilder, ensuring that each config entry carries its + * declared type without runtime type probing or exception handling. + */ +class RecordConfigAccessSuite extends SparkFunSuite { + + // --- configEntryType for builtin entries created via ConfigBuilder --- + + test("configEntryType is BooleanEntry for boolean config entries") { + assert(DRIVER_USER_CLASS_PATH_FIRST.configEntryType === BooleanEntry) + } + + test("configEntryType is IntEntry for int config entries") { + assert(DRIVER_CORES.configEntryType === IntEntry) + } + + test("configEntryType is LongEntry for long config entries") { + assert(STORAGE_UNROLL_MEMORY_THRESHOLD.configEntryType === LongEntry) + } + + test("configEntryType is DoubleEntry for double config entries") { + assert(MEMORY_STORAGE_FRACTION.configEntryType === DoubleEntry) + } + + test("configEntryType is StringEntry for string config entries") { + assert(DRIVER_LIBRARY_PATH.configEntryType === StringEntry) + } + + test("configEntryType is BytesEntry for bytes config entries") { + assert(DRIVER_MEMORY.configEntryType === BytesEntry) + } + + test("configEntryType is TimeEntry for time config entries") { + assert(EXECUTOR_HEARTBEAT_INTERVAL.configEntryType === TimeEntry) + } + + // --- configEntryType propagation through fallbackConf --- + + test("fallbackConf inherits configEntryType from the fallback entry") { + assert(DYN_ALLOCATION_INITIAL_EXECUTORS.configEntryType === IntEntry) + assert(DYN_ALLOCATION_INITIAL_EXECUTORS.configEntryType === + DYN_ALLOCATION_MIN_EXECUTORS.configEntryType) + } + + test("time-typed fallbackConf inherits TimeEntry from fallback") { + assert(DRIVER_METRICS_POLLING_INTERVAL.configEntryType === TimeEntry) + assert(DRIVER_METRICS_POLLING_INTERVAL.configEntryType === + EXECUTOR_HEARTBEAT_INTERVAL.configEntryType) + } + + // --- configEntryType through all create* variants --- + + test("createWithDefault preserves configEntryType") { + val entry = ConfigBuilder("spark.test.createWithDefault.bool") + .booleanConf + .createWithDefault(false) + assert(entry.configEntryType === BooleanEntry) + } + + test("createWithDefaultString preserves configEntryType") { + val entry = ConfigBuilder("spark.test.createWithDefaultString.int") + .intConf + .createWithDefaultString("42") + assert(entry.configEntryType === IntEntry) + } + + test("createWithDefaultFunction preserves configEntryType") { + val entry = ConfigBuilder("spark.test.createWithDefaultFunction.long") + .longConf + .createWithDefaultFunction(() => 100L) + assert(entry.configEntryType === LongEntry) + } + + test("createOptional preserves configEntryType") { + val entry = ConfigBuilder("spark.test.createOptional.double") + .doubleConf + .createOptional + assert(entry.configEntryType === DoubleEntry) + } + + // --- configEntryType survives transform / checkValue / toSequence --- + + test("transform preserves configEntryType") { + val entry = ConfigBuilder("spark.test.transform.int") + .intConf + .transform(v => v * 2) + .createWithDefault(5) + assert(entry.configEntryType === IntEntry) + } + + test("checkValue preserves configEntryType") { + val entry = ConfigBuilder("spark.test.checkValue.double") + .doubleConf + .checkValue(_ > 0, "must be positive") + .createWithDefault(1.0) + assert(entry.configEntryType === DoubleEntry) + } + + test("toSequence preserves configEntryType of element type") { + val entry = ConfigBuilder("spark.test.toSequence.string") + .stringConf + .toSequence + .createWithDefault(Nil) + assert(entry.configEntryType === StringEntry) + } + + // --- each ConfigBuilder.*Conf method assigns the correct enum variant --- + + test("intConf assigns IntEntry") { + val entry = ConfigBuilder("spark.test.intConf") + .intConf + .createWithDefault(0) + assert(entry.configEntryType === IntEntry) + } + + test("longConf assigns LongEntry") { + val entry = ConfigBuilder("spark.test.longConf") + .longConf + .createWithDefault(0L) + assert(entry.configEntryType === LongEntry) + } + + test("doubleConf assigns DoubleEntry") { + val entry = ConfigBuilder("spark.test.doubleConf") + .doubleConf + .createWithDefault(0.0) + assert(entry.configEntryType === DoubleEntry) + } + + test("booleanConf assigns BooleanEntry") { + val entry = ConfigBuilder("spark.test.booleanConf") + .booleanConf + .createWithDefault(false) + assert(entry.configEntryType === BooleanEntry) + } + + test("stringConf assigns StringEntry") { + val entry = ConfigBuilder("spark.test.stringConf") + .stringConf + .createWithDefault("default") + assert(entry.configEntryType === StringEntry) + } + + test("timeConf assigns TimeEntry") { + val entry = ConfigBuilder("spark.test.timeConf") + .timeConf(TimeUnit.SECONDS) + .createWithDefault(10L) + assert(entry.configEntryType === TimeEntry) + } + + test("bytesConf assigns BytesEntry") { + val entry = ConfigBuilder("spark.test.bytesConf") + .bytesConf(ByteUnit.MiB) + .createWithDefault(256L) + assert(entry.configEntryType === BytesEntry) + } + + test("regexConf assigns RegexEntry") { + val entry = ConfigBuilder("spark.test.regexConf") + .regexConf + .createWithDefault(".*".r) + assert(entry.configEntryType === RegexEntry) + } + + // --- non-boolean entries are not BooleanEntry --- + + test("non-boolean config entries do not have BooleanEntry type") { + assert(DRIVER_CORES.configEntryType !== BooleanEntry) + assert(DRIVER_MEMORY.configEntryType !== BooleanEntry) + assert(EXECUTOR_HEARTBEAT_INTERVAL.configEntryType !== BooleanEntry) + assert(STORAGE_UNROLL_MEMORY_THRESHOLD.configEntryType !== BooleanEntry) + assert(MEMORY_STORAGE_FRACTION.configEntryType !== BooleanEntry) + } +}