Skip to content

Commit 79b5b4f

Browse files
committed
fix: hide short URI passwords in ABFS driver warnings
The ABFS resolver skipped secrets under eight characters, so a short password in the URI reached the driver warning when Hadoop named the container-scoped keystore alias it could not recover. URI secrets are now hidden at any length; the floor applies only to configured values. The password is read from the raw authority, split as Hadoop splits the container, so a registry-based authority (an underscore in the host) no longer loses it.
1 parent dbcaa8a commit 79b5b4f

2 files changed

Lines changed: 141 additions & 13 deletions

File tree

‎spark/src/main/scala/org/apache/comet/objectstore/AbfsAuthResolver.scala‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ private[comet] object AbfsAuthResolver extends Logging {
147147
}
148148

149149
private val REDACTED = "<redacted>"
150-
// Shorter values (`true`, a port) would mangle messages without protecting anything.
150+
// Configured values shorter than this (`true`, a port) are not recorded as secrets: hiding them
151+
// would mangle messages without protecting anything. URI secrets are hidden at any length.
151152
private val MIN_REDACTED_LENGTH = 8
152153
// `SimpleKeyProvider` (3.4.1+) puts the raw account key in its message as `key ="..."`.
153154
private val QUOTED_KEY_PATTERN = "key =\"[^\"]*\"".r
@@ -536,15 +537,14 @@ private[comet] object AbfsAuthResolver extends Logging {
536537
if (authType == AuthTypes.SAS) Markers.SAS_PROVIDER_CLASS else Markers.OAUTH_PROVIDER_CLASS
537538

538539
/**
539-
* Hides every `secret` of at least [[MIN_REDACTED_LENGTH]] characters in `message`, and the raw
540-
* key a `SimpleKeyProvider` message quotes. Secrets are matched in the same text, so one that
541-
* overlaps or contains another is hidden whole, and each run of hidden characters becomes one
542-
* marker.
540+
* Hides every non-empty `secret` in `message`, and the raw key a `SimpleKeyProvider` message
541+
* quotes. Secrets are matched in the same text, so one that overlaps or contains another is
542+
* hidden whole, and each run of hidden characters becomes one marker.
543543
*/
544544
private[objectstore] def redact(message: String, secrets: Iterable[String]): String = {
545545
val text = QUOTED_KEY_PATTERN.replaceAllIn(message, "key =\"" + REDACTED + "\"")
546546
val hidden = new Array[Boolean](text.length)
547-
secrets.filter(_.length >= MIN_REDACTED_LENGTH).foreach { secret =>
547+
secrets.filter(_.nonEmpty).foreach { secret =>
548548
var i = text.indexOf(secret)
549549
while (i >= 0) {
550550
Arrays.fill(hidden, i, i + secret.length, true)
@@ -559,10 +559,19 @@ private[comet] object AbfsAuthResolver extends Logging {
559559
out.toString
560560
}
561561

562-
/** The parts of `uri` that can carry a credential: the password and the raw query. */
562+
/**
563+
* The parts of `uri` that can carry a credential: the password and the raw query. The password
564+
* is read from the raw authority, which has one even when `getRawUserInfo` is null (an `_` in
565+
* the host makes the authority registry-based). It is the text after the first `:` of the user
566+
* part, which ends at the last `@` as in [[NativeConfig.authorityWithoutPassword]] and at the
567+
* first `@` as in [[authorityParts]].
568+
*/
563569
private[objectstore] def uriSecrets(uri: URI): Seq[String] = {
564-
val password = Option(uri.getRawUserInfo).map(_.dropWhile(_ != ':').drop(1))
565-
(password ++ Option(uri.getRawQuery)).filter(_.nonEmpty).toSeq
570+
val authority = Option(uri.getRawAuthority).getOrElse("")
571+
val passwords = Seq(authority.lastIndexOf('@'), authority.indexOf('@'))
572+
.filter(_ >= 0)
573+
.map(at => authority.take(at).dropWhile(_ != ':').drop(1))
574+
(passwords ++ Option(uri.getRawQuery)).filter(_.nonEmpty).distinct
566575
}
567576

568577
// Messages reach native errors and logs, so they omit the password, path and query.

‎spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala‎

Lines changed: 123 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@
2020
package org.apache.comet.objectstore
2121

2222
import java.net.URI
23+
import java.nio.charset.StandardCharsets.UTF_8
24+
import java.nio.file.{Files, Path, Paths}
25+
import java.security.KeyStore
26+
import javax.crypto.spec.SecretKeySpec
2327

2428
import scala.collection.mutable.ArrayBuffer
2529
import scala.util.{Failure, Success, Try}
@@ -32,6 +36,11 @@ import org.apache.hadoop.fs.azurebfs.AbfsConfiguration
3236
import org.apache.hadoop.fs.azurebfs.extensions.SASTokenProvider
3337
import org.apache.hadoop.fs.azurebfs.oauth2.{AccessTokenProvider, AzureADToken}
3438
import org.apache.hadoop.fs.azurebfs.services.KeyProvider
39+
import org.apache.hadoop.security.alias.CredentialProviderFactory
40+
import org.apache.logging.log4j.{Level, LogManager}
41+
import org.apache.logging.log4j.core.{LogEvent, Logger => CoreLogger}
42+
import org.apache.logging.log4j.core.appender.AbstractAppender
43+
import org.apache.logging.log4j.core.config.Property
3544

3645
import org.apache.comet.objectstore.AbfsAuthResolver._
3746
import org.apache.comet.objectstore.AbfsReflection.Handles
@@ -77,6 +86,49 @@ class AbfsAuthResolverSuite extends AnyFunSuite with Matchers {
7786
private def cyclic(key: String, other: String): Seq[(String, String)] =
7887
Seq(key -> ("${" + other + "}"), other -> ("${" + key + "}"))
7988

89+
/** A JCEKS keystore Hadoop can open whose one entry, `alias`, it cannot recover. */
90+
private def unrecoverableKeystore(alias: String): Path = {
91+
val storePassword = sys.env.getOrElse("HADOOP_CREDSTORE_PASSWORD", "none").toCharArray
92+
val keyStore = KeyStore.getInstance("JCEKS")
93+
keyStore.load(null, storePassword)
94+
keyStore.setEntry(
95+
alias,
96+
new KeyStore.SecretKeyEntry(new SecretKeySpec("value".getBytes(UTF_8), "AES")),
97+
new KeyStore.PasswordProtection("not-the-store-password".toCharArray))
98+
val tmp = Files.createDirectories(Paths.get(System.getProperty("java.io.tmpdir")))
99+
val path = Files.createTempFile(tmp, "abfs-unrecoverable", ".jceks")
100+
path.toFile.deleteOnExit()
101+
val out = Files.newOutputStream(path)
102+
try keyStore.store(out, storePassword)
103+
finally out.close()
104+
path
105+
}
106+
107+
/** The resolver's WARN messages logged while `f` runs, one per line. */
108+
private def capturedWarnings(f: => Unit): String = {
109+
val logger = LogManager
110+
.getLogger(AbfsAuthResolver.getClass.getName.stripSuffix("$"))
111+
.asInstanceOf[CoreLogger]
112+
val messages = ArrayBuffer[String]()
113+
val appender =
114+
new AbstractAppender("AbfsAuthResolverSuite", null, null, false, Property.EMPTY_ARRAY) {
115+
override def append(event: LogEvent): Unit = messages.synchronized {
116+
if (event.getLevel == Level.WARN) messages += event.getMessage.getFormattedMessage
117+
}
118+
}
119+
appender.start()
120+
val originalLevel = logger.getLevel
121+
logger.addAppender(appender)
122+
logger.setLevel(Level.WARN)
123+
try f
124+
finally {
125+
logger.removeAppender(appender)
126+
logger.setLevel(originalLevel)
127+
appender.stop()
128+
}
129+
messages.synchronized(messages.mkString("\n"))
130+
}
131+
80132
private def assertNoAzureKeys(options: Map[String, String]): Unit = {
81133
val leaked = options.keys.filter(_.startsWith("fs."))
82134
assert(leaked.isEmpty, s"fs.* keys forwarded: $leaked")
@@ -373,6 +425,65 @@ class AbfsAuthResolverSuite extends AnyFunSuite with Matchers {
373425
Seq("the-uri-password", "the-sas-signature").foreach(s => message should not include s)
374426
}
375427

428+
test("Failed - a URI password and query shorter than eight characters are scrubbed too") {
429+
val shortUri = new URI(s"abfss://data:pw12345@$account/p?sig=abc")
430+
val thrown = new ClassNotFoundException(s"no class for $shortUri")
431+
val message = toOptions(resolve(conf(), shortUri, Failure(thrown)))(Markers.ERROR_MESSAGE)
432+
message should include(s"no class for abfss://data:<redacted>@$account/p?<redacted>")
433+
Seq("pw12345", "sig=abc").foreach(s => message should not include s)
434+
}
435+
436+
/**
437+
* The driver warnings from an OAuth client-credentials resolution of a URI on `host` with
438+
* `password` in its user info, where Hadoop cannot read the container-scoped client secret from
439+
* the keystore. The outcome must be a [[Failed]] that does not contain the password.
440+
*/
441+
private def keystoreAliasWarnings(host: String, password: String): String = {
442+
// On 3.4.2+ the container-scoped alias embeds the raw container, user info included, and
443+
// Hadoop names the alias when it cannot recover the entry.
444+
val secretUri = new URI(s"abfss://$container:$password@$host/path/file.parquet")
445+
val keystore = unrecoverableKeystore(s"${Keys.CLIENT_SECRET}.$container:$password.$host")
446+
val providerPath = s"jceks://file${keystore.toUri.getRawPath}"
447+
def key(name: String): String = s"$name.$host"
448+
capturedWarnings {
449+
val outcome = resolve(
450+
conf(
451+
CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH -> providerPath,
452+
key(Keys.AUTH_TYPE) -> "OAuth",
453+
key(Keys.OAUTH_PROVIDER_TYPE) -> Providers.CLIENT_CREDS,
454+
key(Keys.CLIENT_ENDPOINT) -> "https://login.microsoftonline.com/tenant/oauth2/token",
455+
key(Keys.CLIENT_ID) -> "client-123"),
456+
secretUri)
457+
outcome shouldBe a[Failed]
458+
toOptions(outcome).values.foreach(v => v should not include password)
459+
}
460+
}
461+
462+
test("Failed - a short URI password in a keystore alias stays out of the driver warning") {
463+
assume(handles.hasContainerScopedConfiguration, "no container-scoped keys before 3.4.2")
464+
val warnings = keystoreAliasWarnings(account, "pw12345")
465+
warnings should include(s"${Keys.CLIENT_SECRET}.$container:<redacted>.$account")
466+
warnings should not include "pw12345"
467+
}
468+
469+
test("Failed - a URI password is hidden when the host makes the authority registry-based") {
470+
// An underscore in the host leaves java.net.URI without user info, but Hadoop still takes
471+
// the container, password included, from the raw authority.
472+
assume(handles.hasContainerScopedConfiguration, "no container-scoped keys before 3.4.2")
473+
val host = "my_account.dfs.core.windows.net"
474+
assert(new URI(s"abfss://$container:pw12345@$host/p").getRawUserInfo == null)
475+
val warnings = keystoreAliasWarnings(host, "pw12345")
476+
warnings should include(s"${Keys.CLIENT_SECRET}.$container:<redacted>.$host")
477+
warnings should not include "pw12345"
478+
}
479+
480+
test("Failed - a configured value shorter than eight characters is left readable") {
481+
// `true` read as a client secret is too short to hide, so Hadoop's enum error still names it.
482+
val outcome = resolveWith(acct(Keys.AUTH_TYPE) -> "true", acct(Keys.CLIENT_SECRET) -> "true")
483+
outcome shouldBe a[Failed]
484+
outcome.asInstanceOf[Failed].message should include("AuthType.true")
485+
}
486+
376487
test("Failed - a LinkageError from the reflection layer is an error marker, not a crash") {
377488
val outcome = resolve(conf(acct(Keys.ACCOUNT_KEY) -> "c2VjcmV0"), uri, Success(LinkageStub))
378489
outcome shouldBe a[Failed]
@@ -390,10 +501,18 @@ class AbfsAuthResolverSuite extends AnyFunSuite with Matchers {
390501
failed.message should include(AbfsReflection.ClassNames.ABFS_CONFIGURATION)
391502
}
392503

393-
test("redact - hides quoted keys and long secrets, leaves short values alone") {
394-
val message = "Failure for acct key =\"c2VjcmV0c2VjcmV0\": token=the-secret-value port=8080"
395-
redact(message, Seq("the-secret-value", "8080")) shouldBe
396-
"Failure for acct key =\"<redacted>\": token=<redacted> port=8080"
504+
test("uriSecrets - the password comes from the raw authority, at either `@` split") {
505+
uriSecrets(new URI(s"abfss://data:pw@$account/p?sig=x")) shouldBe Seq("pw", "sig=x")
506+
uriSecrets(new URI("abfss://data:pw@my_acct/p")) shouldBe Seq("pw")
507+
uriSecrets(new URI("abfss://data:a@b@acct/p")) shouldBe Seq("a@b", "a")
508+
uriSecrets(new URI(s"abfss://data@$account/p")) shouldBe empty
509+
uriSecrets(new URI("abfss://data:@my_acct/p?")) shouldBe empty
510+
}
511+
512+
test("redact - hides quoted keys and every non-empty secret, however short") {
513+
val message = "Failure for acct key =\"c2VjcmV0c2VjcmV0\": token=the-secret-value pw=1234567"
514+
redact(message, Seq("the-secret-value", "1234567", "")) shouldBe
515+
"Failure for acct key =\"<redacted>\": token=<redacted> pw=<redacted>"
397516
}
398517

399518
test("redact - overlapping, contained and adjacent secrets become one marker") {

0 commit comments

Comments
 (0)