Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Delta Lake checkpoint cleanups #21195

Merged
merged 8 commits into from
Mar 22, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Remove unnecessary usage of BigInteger for LastCheckpoint
  • Loading branch information
electrum committed Mar 22, 2024
commit 11b395e7c313b73fe4e916be384c41d51c457c0d
Original file line number Diff line number Diff line change
@@ -20,7 +20,6 @@
import io.trino.plugin.deltalake.transactionlog.RemoveFileEntry;
import io.trino.plugin.deltalake.transactionlog.TransactionEntry;

import java.math.BigInteger;
import java.util.Objects;
import java.util.Set;
import java.util.StringJoiner;
@@ -74,10 +73,10 @@ public Set<RemoveFileEntry> getRemoveFileEntries()
return removeFileEntries;
}

public BigInteger size()
public long size()
{
// The additional 2 are for the MetadataEntry and ProtocolEntry
return BigInteger.valueOf(transactionEntries.size() + addFileEntries.size() + removeFileEntries.size() + 2);
return transactionEntries.size() + addFileEntries.size() + removeFileEntries.size() + 2;
}

@Override
Original file line number Diff line number Diff line change
@@ -16,7 +16,6 @@
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

import java.math.BigInteger;
import java.util.Objects;
import java.util.Optional;

@@ -26,17 +25,17 @@
public class LastCheckpoint
{
private final long version;
private final BigInteger size;
private final long size;
private final Optional<Integer> parts;

@JsonCreator
public LastCheckpoint(
@JsonProperty("version") long version,
@JsonProperty("size") BigInteger size,
@JsonProperty("size") long size,
@JsonProperty("parts") Optional<Integer> parts)
{
this.version = version;
this.size = requireNonNull(size, "size is null");
this.size = size;
this.parts = requireNonNull(parts, "parts is null");
}

@@ -47,7 +46,7 @@ public long getVersion()
}

@JsonProperty
public BigInteger getSize()
public long getSize()
{
return size;
}
@@ -69,7 +68,7 @@ public boolean equals(Object o)
}
LastCheckpoint that = (LastCheckpoint) o;
return version == that.version &&
size.equals(that.size) &&
size == that.size &&
parts.equals(that.parts);
}

Original file line number Diff line number Diff line change
@@ -25,7 +25,6 @@
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.math.BigInteger;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.Arrays;
@@ -87,7 +86,7 @@ public void testReadLastCheckpointFile()
{
LastCheckpoint lastCheckpoint = objectMapper.readValue("{\"version\":10,\"size\":17}", LastCheckpoint.class);
assertThat(lastCheckpoint.getVersion()).isEqualTo(10L);
assertThat(lastCheckpoint.getSize()).isEqualTo(BigInteger.valueOf(17L));
assertThat(lastCheckpoint.getSize()).isEqualTo(17);
assertThat(lastCheckpoint.getParts()).isEqualTo(Optional.empty());
}

@@ -97,7 +96,7 @@ public void testReadLastCheckpointFileForMultipart()
{
LastCheckpoint lastCheckpoint = objectMapper.readValue("{\"version\":237580,\"size\":658573,\"parts\":2}", LastCheckpoint.class);
assertThat(lastCheckpoint.getVersion()).isEqualTo(237580L);
assertThat(lastCheckpoint.getSize()).isEqualTo(BigInteger.valueOf(658573L));
assertThat(lastCheckpoint.getSize()).isEqualTo(658573L);
assertThat(lastCheckpoint.getParts()).isEqualTo(Optional.of(2));
}