Skip to content

Commit

Permalink
Add Alluxio file system support
Browse files Browse the repository at this point in the history
  • Loading branch information
JiamingMai authored and raunaqmorarka committed Sep 27, 2024
1 parent 3468390 commit 44178ea
Show file tree
Hide file tree
Showing 19 changed files with 1,712 additions and 28 deletions.
125 changes: 125 additions & 0 deletions lib/trino-filesystem-alluxio/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>io.trino</groupId>
<artifactId>trino-root</artifactId>
<version>460-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<artifactId>trino-filesystem-alluxio</artifactId>
<description>Trino Filesystem - Alluxio</description>

<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<air.compiler.fail-warnings>true</air.compiler.fail-warnings>
</properties>

<dependencies>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-filesystem</artifactId>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-memory-context</artifactId>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-spi</artifactId>
</dependency>

<dependency>
<groupId>org.alluxio</groupId>
<artifactId>alluxio-core-client-fs</artifactId>
</dependency>

<dependency>
<groupId>org.alluxio</groupId>
<artifactId>alluxio-core-common</artifactId>
</dependency>

<dependency>
<groupId>org.alluxio</groupId>
<artifactId>alluxio-core-transport</artifactId>
</dependency>

<dependency>
<groupId>com.github.docker-java</groupId>
<artifactId>docker-java-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>junit-extensions</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-filesystem</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-spi</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-testing</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.trino</groupId>
<artifactId>trino-testing-containers</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.filesystem.alluxio;

import alluxio.client.file.URIStatus;
import io.trino.filesystem.FileEntry;
import io.trino.filesystem.FileIterator;
import io.trino.filesystem.Location;

import java.io.IOException;
import java.time.Instant;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;

import static io.trino.filesystem.alluxio.AlluxioUtils.convertToLocation;
import static java.util.Objects.requireNonNull;

public class AlluxioFileIterator
implements FileIterator
{
private final Iterator<URIStatus> files;
private final String mountRoot;

public AlluxioFileIterator(List<URIStatus> files, String mountRoot)
{
this.files = requireNonNull(files.iterator(), "files is null");
this.mountRoot = requireNonNull(mountRoot, "mountRoot is null");
}

@Override
public boolean hasNext()
throws IOException
{
return files.hasNext();
}

@Override
public FileEntry next()
throws IOException
{
if (!hasNext()) {
return null;
}
URIStatus fileStatus = files.next();
Location location = convertToLocation(fileStatus, mountRoot);
return new FileEntry(
location,
fileStatus.getLength(),
Instant.ofEpochMilli(fileStatus.getLastModificationTimeMs()),
Optional.empty());
}
}
Loading

0 comments on commit 44178ea

Please sign in to comment.