Skip to content

[filesystem] Add JuiceFS filesystem support #3870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
weizuo93 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

[filesystem] Add JuiceFS filesystem support #3870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c84b1e
[filesystem] Add JuiceFS filesystem support
May 18, 2026
043264d
[filesystem] Fix JuiceFS docs and log
Aug 12, 2026
9a4cd57
[filesystem] Fix JuiceFS SecurityToken and add unit test
Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 193 additions & 0 deletions fluss-filesystems/fluss-fs-juicefs/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.fluss</groupId>
<artifactId>fluss-filesystems</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>

<artifactId>fluss-fs-juicefs</artifactId>
<name>Fluss : FileSystems : JuiceFS</name>

<properties>
<fs.juicefs.sdk.version>1.4.0</fs.juicefs.sdk.version>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.fluss</groupId>
<artifactId>fluss-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.fluss</groupId>
<artifactId>fluss-fs-hadoop-shaded</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.fluss</groupId>
<artifactId>fluss-fs-hadoop</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>io.juicefs</groupId>
<artifactId>juicefs-hadoop</artifactId>
<version>${fs.juicefs.sdk.version}</version>
<exclusions>
<!-- provided by fluss-fs-hadoop-shaded -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<!-- avoid logger conflicts -->
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- test dependencies -->
<dependency>
<groupId>org.apache.fluss</groupId>
<artifactId>fluss-test-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.fluss</groupId>
<artifactId>fluss-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<type>test-jar</type>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<id>shade-fluss</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<includes>
<include>*:*</include>
</includes>
<excludes>
<exclude>javax.servlet:servlet-api</exclude>
<exclude>xmlenc:xmlenc</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*</artifact>
<excludes>
<exclude>.gitkeep</exclude>
<exclude>mime.types</exclude>
<exclude>mozilla/**</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>license/LICENSE*</exclude>
<exclude>NOTICE</exclude>
</excludes>
</filter>
<filter>
<artifact>org.apache.fluss:fluss-fs-hadoop</artifact>
<excludes>
<exclude>META-INF/**</exclude>
</excludes>
</filter>
<filter>
<!--
juicefs-hadoop is an uber jar that bundles Sun Jersey 1.x
(CDDL/GPLv2 dual-licensed) and its metadata files. Fluss
does not use Jersey; strip these classes and resources to
keep the shaded artifact free of GPL-flagged content and
avoid conflicting META-INF headers from the uber jar.
-->
<artifact>io.juicefs:juicefs-hadoop</artifact>
<excludes>
<exclude>com/sun/jersey/**</exclude>
<exclude>META-INF/maven/com.sun.jersey/**</exclude>
<exclude>META-INF/taglib.tld</exclude>
<exclude>META-INF/LICENSE</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>META-INF/DEPENDENCIES</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org.apache.commons</pattern>
<shadedPattern>org.apache.fluss.shaded.org.apache.commons</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
50 changes: 50 additions & 0 deletions ...systems/fluss-fs-juicefs/src/main/java/org/apache/fluss/fs/juicefs/JuiceFsFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.fluss.fs.juicefs;

import org.apache.fluss.fs.hdfs.HadoopFileSystem;
import org.apache.fluss.fs.token.ObtainedSecurityToken;

import org.apache.hadoop.fs.FileSystem;

import java.util.Collections;

/**
* A {@link org.apache.fluss.fs.FileSystem} for JuiceFS that wraps a {@link HadoopFileSystem}.
*
* <p>Unlike the OSS / S3 plugins, JuiceFS does not require Fluss to obtain or distribute a
* delegation token: the JuiceFS client itself authenticates against the meta server using
* locally-configured credentials (e.g. {@code juicefs.access-key} / {@code juicefs.secret-key}, or
* implicit IAM). We therefore return an empty placeholder token from {@link
* #obtainSecurityToken()}.
*/
class JuiceFsFileSystem extends HadoopFileSystem {

private static final ObtainedSecurityToken EMPTY_TOKEN =
new ObtainedSecurityToken(
JuiceFsPlugin.SCHEME, new byte[0], null, Collections.emptyMap());

JuiceFsFileSystem(FileSystem hadoopFileSystem) {
super(hadoopFileSystem);
}

@Override
public ObtainedSecurityToken obtainSecurityToken() {
return EMPTY_TOKEN;

@luoyuxia luoyuxia Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

obtainSecurityToken() returns a non-null token with scheme jfs, so DefaultSecurityTokenManager always forwards it to SecurityTokenReceiverRepository. This module does not register a jfs receiver, causing every client to log no receiver found and schedule another token refresh after the retry backoff. Could we add a no-op receiver and SPI entry, similar to HDFS, or explicitly skip receiver notification for placeholder tokens? A small regression test for this path would also help.

@weizuo93 weizuo93 Aug 13, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK

}
}
147 changes: 147 additions & 0 deletions ...filesystems/fluss-fs-juicefs/src/main/java/org/apache/fluss/fs/juicefs/JuiceFsPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.fluss.fs.juicefs;

import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.config.ConfigBuilder;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.fs.FileSystem;
import org.apache.fluss.fs.FileSystemPlugin;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;

/**
* Simple factory for the JuiceFS file system.
*
* <p>This plugin registers the {@code jfs} scheme and bridges Fluss to the JuiceFS Hadoop SDK
* ({@code io.juicefs.JuiceFileSystem}). The JuiceFS client itself manages authentication via its
* meta server / access keys, so no Fluss-side delegation token is required.
*
* <p>Configuration is propagated from Fluss to the underlying Hadoop {@link
* org.apache.hadoop.conf.Configuration} for any key starting with {@code fs.jfs.} or {@code
* juicefs.}. Two defaults are injected when not provided by the user:
*
* <ul>
* <li>{@code fs.jfs.impl=io.juicefs.JuiceFileSystem}
* <li>{@code fs.jfs.impl.disable.cache=false}
* </ul>
*/
public class JuiceFsPlugin implements FileSystemPlugin {

private static final Logger LOG = LoggerFactory.getLogger(JuiceFsPlugin.class);

public static final String SCHEME = "jfs";

/** Fully qualified class name of the JuiceFS Hadoop SDK FileSystem implementation. */
static final String JUICEFS_HADOOP_FS_IMPL = "io.juicefs.JuiceFileSystem";

/** Hadoop configuration key that selects the {@code jfs} scheme implementation. */
static final String FS_JFS_IMPL_KEY = "fs.jfs.impl";

/** Hadoop configuration key controlling the FileSystem cache for the {@code jfs} scheme. */
static final String FS_JFS_IMPL_DISABLE_CACHE_KEY = "fs.jfs.impl.disable.cache";

/**
* In order to simplify, fluss juicefs configuration keys mirror the upstream Hadoop / JuiceFS
* keys. Any Fluss config entry whose key starts with one of these prefixes is forwarded to the
* Hadoop configuration.
*
* <ul>
* <li>{@code fs.jfs.} — Hadoop FileSystem-framework level keys (impl class, cache, etc.)
* <li>{@code juicefs.} — native JuiceFS client knobs (meta URL, cache-dir, access-key, ...)
* </ul>
*/
private static final String[] FLUSS_CONFIG_PREFIXES = {"fs.jfs.", "juicefs."};

@Override
public String getScheme() {
return SCHEME;
}

@Override
public FileSystem create(URI fsUri, Configuration flussConfig) throws IOException {
org.apache.hadoop.conf.Configuration hadoopConfig = getHadoopConfiguration(flussConfig);
applyJuiceFsDefaults(hadoopConfig);

// handle missing scheme/authority by falling back to the configured default URI
final String scheme = fsUri.getScheme();
final String authority = fsUri.getAuthority();
if (scheme == null && authority == null) {
fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
} else if (scheme != null && authority == null) {
URI defaultUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig);
if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) {
fsUri = defaultUri;
}
}

// load the JuiceFS Hadoop SDK reflectively via Hadoop's FileSystem.newInstance, so that
// the Fluss bytecode does not have a hard compile-time dependency on io.juicefs classes
org.apache.hadoop.fs.FileSystem hadoopFs =
org.apache.hadoop.fs.FileSystem.newInstance(fsUri, hadoopConfig);
LOG.info(
"Created JuiceFS Hadoop FileSystem: scheme={}, authority={}, impl={}",
fsUri.getScheme(),
fsUri.getAuthority(),
hadoopFs.getClass().getName());

return new JuiceFsFileSystem(hadoopFs);
}

/**
* Inject sensible defaults for the JuiceFS Hadoop bridge if the user has not specified them.
*
* <p>This is package-private to allow direct testing without a live JuiceFS meta server.
*/
@VisibleForTesting
static void applyJuiceFsDefaults(org.apache.hadoop.conf.Configuration hadoopConfig) {
if (hadoopConfig.get(FS_JFS_IMPL_KEY) == null) {
hadoopConfig.set(FS_JFS_IMPL_KEY, JUICEFS_HADOOP_FS_IMPL);
}
if (hadoopConfig.get(FS_JFS_IMPL_DISABLE_CACHE_KEY) == null) {
hadoopConfig.set(FS_JFS_IMPL_DISABLE_CACHE_KEY, "false");
}
}

@VisibleForTesting
org.apache.hadoop.conf.Configuration getHadoopConfiguration(Configuration flussConfig) {
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
if (flussConfig == null) {
return conf;
}

// read all configuration entries with a prefix in 'FLUSS_CONFIG_PREFIXES'
for (String key : flussConfig.keySet()) {
for (String prefix : FLUSS_CONFIG_PREFIXES) {
if (key.startsWith(prefix)) {
String value =
flussConfig.getString(
ConfigBuilder.key(key).stringType().noDefaultValue(), null);
conf.set(key, value);
LOG.debug("Adding Fluss config entry {} to Hadoop config", key);
break;
}
}
}
return conf;
}
}
Loading
Loading