From 3e520f1fef3b79b488088c339bddc7bd321d333d Mon Sep 17 00:00:00 2001 From: jja725 Date: Fri, 26 Apr 2024 11:41:48 -0700 Subject: [PATCH] Update local cache to catch 3.x ### What changes are proposed in this pull request? cherry pick all local cache related stuff back to 2.x. The principle here is to keep Trino using the same code as 3.x ### Why are the changes needed? Please clarify why the changes are needed. For instance, 1. If you propose a new API, clarify the use case for a new API. 2. If you fix a bug, describe the bug. ### Does this PR introduce any user facing changes? Please list the user-facing changes introduced by your change, including 1. change in user-facing APIs 2. addition or removal of property keys 3. webui pr-link: Alluxio/alluxio#18574 change-id: cid-3e548ccfbfd290dcefd978a55221a92753f99118 --- .../client/file/cache/CacheManager.java | 94 +++- .../cache/CacheManagerWithShadowCache.java | 48 +- .../client/file/cache/CacheStatus.java | 26 + .../alluxio/client/file/cache/CacheUsage.java | 249 +++++++++ .../client/file/cache/CacheUsageView.java | 119 +++++ .../file/cache/DefaultPageMetaStore.java | 85 ++- .../file/cache/LocalCacheFileInStream.java | 108 ++-- .../file/cache/LocalCacheFileSystem.java | 4 +- .../client/file/cache/LocalCacheManager.java | 501 +++++++++++++++--- .../file/cache/NoExceptionCacheManager.java | 96 +++- .../alluxio/client/file/cache/PageInfo.java | 1 + .../client/file/cache/PageMetaStore.java | 31 +- .../alluxio/client/file/cache/PageStore.java | 48 +- .../client/file/cache/QuotaPageMetaStore.java | 20 + .../client/file/cache/TimeBoundPageStore.java | 11 +- .../cache/context/CachePerThreadContext.java | 43 ++ .../file/cache/store/LocalPageStore.java | 96 +++- .../file/cache/store/LocalPageStoreDir.java | 39 +- .../file/cache/store/MemoryPageStore.java | 26 +- .../file/cache/store/MemoryPageStoreDir.java | 6 + .../client/file/cache/store/PageStoreDir.java | 14 +- .../cache/store/QuotaManagedPageStoreDir.java | 74 ++- .../file/cache/store/RocksPageStore.java | 3 +- .../file/cache/store/RocksPageStoreDir.java | 6 + .../alluxio/client/file/MockFileInStream.java | 2 +- .../CacheManagerWithShadowCacheTest.java | 53 +- .../client/file/cache/HangingPageStore.java | 4 +- .../cache/LocalCacheFileInStreamTest.java | 208 +++++++- .../file/cache/LocalCacheManagerTest.java | 146 ++++- .../file/cache/TimeBoundPageStoreTest.java | 2 +- .../file/cache/store/LocalPageStoreTest.java | 19 +- .../file/cache/store/MemoryPageStoreTest.java | 4 +- .../file/cache/store/PageStoreDirTest.java | 35 +- .../file/cache/store/PageStoreTest.java | 11 +- .../alluxio/hadoop/LocalCacheFileSystem.java | 35 +- core/common/pom.xml | 20 + .../main/java/alluxio/conf/PropertyKey.java | 9 + .../exception/PageCorruptedException.java | 35 ++ .../alluxio/file/ByteArrayTargetBuffer.java | 105 ++++ .../alluxio/file/ByteBufferTargetBuffer.java | 114 ++++ .../alluxio/file/NettyBufTargetBuffer.java | 120 +++++ .../java/alluxio/file/ReadTargetBuffer.java | 86 +++ .../main/java/alluxio/metrics/MetricKey.java | 30 ++ .../MultiDimensionalMetricsSystem.java | 311 +++++++++++ .../protocol/databuffer/DataFileChannel.java | 86 +++ .../java/alluxio/util/io/ChannelAdapters.java | 141 +++++ .../worker/page/PagedBlockMetaStore.java | 102 ++-- .../alluxio/worker/page/PagedBlockReader.java | 5 +- .../worker/page/PagedBlockStoreDir.java | 10 + .../worker/page/ByteArrayCacheManager.java | 76 ++- .../worker/page/PagedBlockWriterTest.java | 2 +- pom.xml | 21 + .../fs/LocalCacheManagerIntegrationTest.java | 48 +- 53 files changed, 3291 insertions(+), 297 deletions(-) create mode 100644 core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java create mode 100644 core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java create mode 100644 core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java create mode 100644 core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java create mode 100644 core/common/src/main/java/alluxio/exception/PageCorruptedException.java create mode 100644 core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java create mode 100644 core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java create mode 100644 core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java create mode 100644 core/common/src/main/java/alluxio/file/ReadTargetBuffer.java create mode 100644 core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java create mode 100644 core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java create mode 100644 core/common/src/main/java/alluxio/util/io/ChannelAdapters.java diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java index 2ca67855a047..1269d0e68338 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java @@ -12,12 +12,15 @@ package alluxio.client.file.cache; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.PropertyKey; +import alluxio.exception.PageNotFoundException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.metrics.MultiDimensionalMetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.resource.LockResource; import com.codahale.metrics.Counter; @@ -27,16 +30,19 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.function.Supplier; import javax.annotation.concurrent.GuardedBy; /** * Interface for managing cached pages. */ -public interface CacheManager extends AutoCloseable { +public interface CacheManager extends AutoCloseable, CacheStatus { + Logger LOG = LoggerFactory.getLogger(CacheManager.class); /** * State of a cache. @@ -119,6 +125,12 @@ public static CacheManager create(AlluxioConfiguration conf, try { boolean isShadowCacheEnabled = conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_SHADOW_ENABLED); + boolean isNettyDataTransmissionEnable = false; + // Note that Netty data transmission doesn't support async write + if (isNettyDataTransmissionEnable) { + options.setIsAsyncWriteEnabled(false); + } + MultiDimensionalMetricsSystem.setCacheStorageSupplier(pageMetaStore::bytes); if (isShadowCacheEnabled) { return new NoExceptionCacheManager( new CacheManagerWithShadowCache(LocalCacheManager.create(options, pageMetaStore), @@ -140,6 +152,8 @@ static void clear() { CacheManager manager = CACHE_MANAGER.getAndSet(null); if (manager != null) { manager.close(); + MultiDimensionalMetricsSystem.setCacheStorageSupplier( + MultiDimensionalMetricsSystem.NULL_SUPPLIER); } } catch (Exception e) { LOG.warn("Failed to close CacheManager: {}", e.toString()); @@ -236,6 +250,20 @@ default int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, return get(pageId, pageOffset, bytesToRead, buffer, offsetInBuffer, CacheContext.defaults()); } + /** + * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. + * + * @param pageId page identifier + * @param pageOffset offset into the page + * @param buffer destination buffer to write + * @param cacheContext cache related context + * @return number of bytes read, 0 if page is not found, -1 on errors + */ + default int get(PageId pageId, int pageOffset, ReadTargetBuffer buffer, + CacheContext cacheContext) { + throw new UnsupportedOperationException("This method is unsupported. "); + } + /** * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. * @@ -263,9 +291,24 @@ default int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, i * @param cacheContext cache related context * @return number of bytes read, 0 if page is not found, -1 on errors */ - int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, CacheContext cacheContext); + /** + * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. + * Loads the page otherwise. + * + * @param pageId page identifier + * @param pageOffset offset into the page + * @param bytesToRead number of bytes to read in this page + * @param buffer destination buffer to write + * @param cacheContext cache related context + * @param externalDataSupplier the external data supplier to read a page + * @return number of bytes read, 0 if page is not found, -1 on errors + */ + int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, Supplier externalDataSupplier); + /** * Get page ids by the given file id. * @param fileId file identifier @@ -276,6 +319,20 @@ default List getCachedPageIdsByFileId(String fileId, long fileLength) { throw new UnsupportedOperationException(); } + /** + * Deletes all pages of the given file. + * + * @param fileId the file id of the target file + */ + void deleteFile(String fileId); + + /** + * Deletes all temporary pages of the given file. + * + * @param fileId the file id of the target file + */ + void deleteTempFile(String fileId); + /** * Deletes a page from the cache. * @@ -289,6 +346,14 @@ default List getCachedPageIdsByFileId(String fileId, long fileLength) { */ State state(); + /** + * @param pageId the page id + * @return true if the page is cached. This method is not thread-safe as no lock is acquired + */ + default boolean hasPageUnsafe(PageId pageId) { + throw new UnsupportedOperationException(); + } + /** * * @param pageId @@ -306,4 +371,25 @@ default List getCachedPageIdsByFileId(String fileId, long fileLength) { default void invalidate(Predicate predicate) { throw new UnsupportedOperationException(); } + + @Override + Optional getUsage(); + + /** + * Commit the File. + * @param fileId the file ID + */ + void commitFile(String fileId); + + /** + * Get a {@link DataFileChannel} which wraps a {@link io.netty.channel.FileRegion}. + * @param pageId the page id + * @param pageOffset the offset inside the page + * @param bytesToRead the bytes to read + * @param cacheContext the cache context + * @return an object of {@link DataFileChannel} + */ + Optional getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, CacheContext cacheContext) + throws PageNotFoundException; } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java index e6c98b2306f4..d2039423ad03 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java @@ -14,11 +14,13 @@ import static alluxio.client.file.CacheContext.StatsUnit.BYTE; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.quota.CacheScope; import alluxio.conf.AlluxioConfiguration; +import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; @@ -26,6 +28,8 @@ import com.google.common.hash.PrimitiveSink; import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.function.Supplier; import javax.annotation.Nonnull; /** @@ -46,6 +50,11 @@ public CacheManagerWithShadowCache(CacheManager cacheManager, AlluxioConfigurati mShadowCacheManager = ShadowCacheManager.create(conf); } + @Override + public void commitFile(String fileId) { + mCacheManager.commitFile(fileId); + } + @Override public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { updateShadowCache(pageId, page.remaining(), cacheContext); @@ -53,8 +62,21 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { + getOrUpdateShadowCache(pageId, bytesToRead, cacheContext); + return mCacheManager.get(pageId, pageOffset, bytesToRead, target, cacheContext); + } + + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, Supplier externalDataSupplier) { + getOrUpdateShadowCache(pageId, bytesToRead, cacheContext); + return mCacheManager.getAndLoad(pageId, pageOffset, bytesToRead, + buffer, cacheContext, externalDataSupplier); + } + + private void getOrUpdateShadowCache(PageId pageId, int bytesToRead, CacheContext cacheContext) { int nread = mShadowCacheManager.get(pageId, bytesToRead, getCacheScope(cacheContext)); if (nread > 0) { Metrics.SHADOW_CACHE_PAGES_HIT.inc(); @@ -64,7 +86,6 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf } Metrics.SHADOW_CACHE_PAGES_READ.inc(); Metrics.SHADOW_CACHE_BYTES_READ.inc(bytesToRead); - return mCacheManager.get(pageId, pageOffset, bytesToRead, target, cacheContext); } /** @@ -142,6 +163,27 @@ public void close() throws Exception { mCacheManager.close(); } + @Override + public void deleteFile(String fileId) { + mCacheManager.deleteFile(fileId); + } + + @Override + public void deleteTempFile(String fileId) { + mCacheManager.deleteTempFile(fileId); + } + + @Override + public Optional getUsage() { + return mCacheManager.getUsage(); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) throws PageNotFoundException { + return mCacheManager.getDataFileChannel(pageId, pageOffset, bytesToRead, cacheContext); + } + /** * Decrease each item's clock and clean stale items. */ diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java new file mode 100644 index 000000000000..1f5b3c5ae2e8 --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java @@ -0,0 +1,26 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache; + +import java.util.Optional; + +/** + * Mixin interface for various cache status info. + */ +public interface CacheStatus { + /** + * Gets cache usage. + * + * @return cache usage, or none if reporting usage info is not supported + */ + Optional getUsage(); +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java new file mode 100644 index 000000000000..29f1f72da15f --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java @@ -0,0 +1,249 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache; + +import alluxio.client.quota.CacheScope; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; +import java.util.Optional; + +/** + * Cache usage. + *
+ * Granularity + * A usage object of this interface is associated with a certain granularity, either global, + * of a cache directory, of a application-defined scope, or of a particular file. Coarse-grained + * cache usage objects may be partitioned into a finer-grained one, for example the global usage + * can be partitioned into usages of each cache directory. + *
+ * Stats + * The following cache usage stats are reported for the granularity of the object: + *
    + *
  • Used: size of pages currently cached
    • + *
  • Available: size of free space, plus size of evictable pages
    • + *
  • Capacity: total capacity
    • + *
+ * Snapshot + * Cache usage object does not offer atomic view of their stats. Two subsequent calls to get the + * same stats may return different results, as the underlying cached contents may have changed + * between the calls. To get a snapshot of the cache stats, call {@link #snapshot()} to obtain + * an immutable view of the cache usage. + */ +//todo(bowen): allow introspection of the granularity of the cache usage object +public interface CacheUsage extends CacheUsageView { + /** + * Creates an immutable snapshot of the current cache usage. + * + * @return the snapshot + */ + default CacheUsageView snapshot() { + return new ImmutableCacheUsageView(used(), available(), capacity()); + } + + /** + * Gets a finer-grained view of cache usage. + *
Example of getting cache usage of a particular file: + *
+ * 
{@code
+   * cacheManager.getUsage()
+   *     .flatMap(usage -> usage.partitionedBy(PartitionDescriptor.file(fileId))
+   *     .map(CacheUsage::used)
+   *     .orElse(0)
+   * }
+ * + * @param partition how to partition the cache + * @return partitioned view of cache usage, none if this usage object does not support + * partitioning, or usage info for the specified partition is not found + */ + Optional partitionedBy(PartitionDescriptor partition); + + /** + * Partition descriptor. + * + * @param type of the identifier + */ + interface PartitionDescriptor { + /** + * Gets an identifier of the partition. + * + * @return identifier + */ + T getIdentifier(); + + /** + * Creates a partition for a specific file. + * + * @param fileId file ID + * @return the partition + */ + static FilePartition file(String fileId) { + return new FilePartition(fileId); + } + + /** + * Creates a partition for a directory. + * + * @param index dir index + * @return the partition + */ + static DirPartition dir(int index) { + return new DirPartition(index); + } + + /** + * Creates a partition of a cache scope. + * + * @param scope the cache scope + * @return the partition + */ + static ScopePartition scope(CacheScope scope) { + return new ScopePartition(scope); + } + } + + /** + * Partition on a particular cache directory. + */ + final class DirPartition implements PartitionDescriptor { + private final int mIndex; + + /** + * Creates a partition based on the directory's index. + * + * @param index dir index + */ + public DirPartition(int index) { + mIndex = index; + } + + @Override + public Integer getIdentifier() { + return mIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DirPartition that = (DirPartition) o; + return mIndex == that.mIndex; + } + + @Override + public int hashCode() { + return Objects.hash(mIndex); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("dirIndex", mIndex) + .toString(); + } + } + + /** + * Partition on a cache scope. + */ + final class ScopePartition implements PartitionDescriptor { + private final CacheScope mScope; + + /** + * Creates a partition over a cache scope. + * + * @param scope cache scope + */ + public ScopePartition(CacheScope scope) { + mScope = scope; + } + + @Override + public CacheScope getIdentifier() { + return mScope; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ScopePartition that = (ScopePartition) o; + return Objects.equals(mScope, that.mScope); + } + + @Override + public int hashCode() { + return Objects.hash(mScope); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("scope", mScope) + .toString(); + } + } + + /** + * Partition on a particular file. + */ + final class FilePartition implements PartitionDescriptor { + private final String mFileId; + + /** + * Creates a partition over a file ID. + * + * @param fileId the file ID + */ + public FilePartition(String fileId) { + mFileId = fileId; + } + + @Override + public String getIdentifier() { + return mFileId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FilePartition that = (FilePartition) o; + return Objects.equals(mFileId, that.mFileId); + } + + @Override + public int hashCode() { + return Objects.hash(mFileId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("fileId", mFileId) + .toString(); + } + } +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java new file mode 100644 index 000000000000..45c2ced3c5d2 --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java @@ -0,0 +1,119 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; +import java.util.Optional; + +/** + * Immutable view of cache usage stats. + */ +public interface CacheUsageView { + /** + * Bytes currently used by the cache. + * This includes both evictable and inevitable pages. + * + * @return number of bytes being used + */ + long used(); + + /** + * Bytes that are available for caching. + * This includes free space, as well as the space occupied by evictable pages. + * + * @return number of bytes that can be used for caching + */ + long available(); + + /** + * Total capacity of the cache. + * @return total number of bytes the cache is allowed to use + */ + long capacity(); + + /** + * Immutable holder of cache stats. + */ + final class ImmutableCacheUsageView implements CacheUsage { + private final long mUsed; + private final long mAvailable; + private final long mCapacity; + + /** + * Constructor. + * + * @param used used + * @param available available + * @param capacity capacity + */ + public ImmutableCacheUsageView(long used, long available, long capacity) { + mUsed = used; + mAvailable = available; + mCapacity = capacity; + } + + @Override + public long used() { + return mUsed; + } + + @Override + public long available() { + return mAvailable; + } + + @Override + public long capacity() { + return mCapacity; + } + + @Override + public CacheUsageView snapshot() { + return this; + } + + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ImmutableCacheUsageView that = (ImmutableCacheUsageView) o; + return mUsed == that.mUsed + && mAvailable == that.mAvailable + && mCapacity == that.mCapacity; + } + + @Override + public int hashCode() { + return Objects.hash(mUsed, mAvailable, mCapacity); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("used", mUsed) + .add("available", mAvailable) + .add("capacity", mCapacity) + .toString(); + } + } +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java index 1c8e2b748119..daa12da40dc3 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java @@ -20,6 +20,7 @@ import alluxio.client.quota.CacheScope; import alluxio.collections.IndexDefinition; import alluxio.collections.IndexedSet; +import alluxio.exception.FileDoesNotExistException; import alluxio.exception.PageNotFoundException; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; @@ -31,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -121,15 +123,24 @@ public void commitFile(String fileId, String newFileId) throws PageNotFoundExcep throw new PageNotFoundException( String.format("No Pages found for file %s when committing", fileId)); } - for (PageInfo oldPage : pages) { - PageId newPageId = new PageId(newFileId, oldPage.getPageId().getPageIndex()); - PageInfo newPageInfo = new PageInfo(newPageId, oldPage.getPageSize(), oldPage.getScope(), - oldPage.getLocalCacheDir()); - mPages.remove(oldPage); + for (PageInfo oldPageInfo : pages) { + PageId newPageId = new PageId(newFileId, oldPageInfo.getPageId().getPageIndex()); + PageInfo newPageInfo = new PageInfo(newPageId, oldPageInfo.getPageSize(), + oldPageInfo.getScope(), oldPageInfo.getLocalCacheDir()); + mPages.remove(oldPageInfo); mPages.add(newPageInfo); } } + @Override + public PageStoreDir getStoreDirOfFile(String fileId) throws FileDoesNotExistException { + PageInfo pageInfo = mPages.getFirstByField(INDEX_FILE_ID, fileId); + if (pageInfo == null) { + throw new FileDoesNotExistException(String.format("File %s does not exist in cache", fileId)); + } + return pageInfo.getLocalCacheDir(); + } + @Override public List getStoreDirs() { return mDirs; @@ -153,7 +164,7 @@ public PageInfo getPageInfo(PageId pageId) throws PageNotFoundException { @Override @GuardedBy("getLock()") - public PageInfo removePage(PageId pageId) throws PageNotFoundException { + public PageInfo removePage(PageId pageId, boolean isTemporary) throws PageNotFoundException { if (!mPages.contains(INDEX_PAGE_ID, pageId)) { throw new PageNotFoundException(String.format("Page %s could not be found", pageId)); } @@ -162,10 +173,20 @@ public PageInfo removePage(PageId pageId) throws PageNotFoundException { mPages.remove(pageInfo); mBytes.addAndGet(-pageInfo.getPageSize()); Metrics.SPACE_USED.dec(pageInfo.getPageSize()); - pageInfo.getLocalCacheDir().deletePage(pageInfo); + if (isTemporary) { + pageInfo.getLocalCacheDir().deleteTempPage(pageInfo); + } else { + pageInfo.getLocalCacheDir().deletePage(pageInfo); + } return pageInfo; } + @Override + @GuardedBy("getLock()") + public PageInfo removePage(PageId pageId) throws PageNotFoundException { + return removePage(pageId, false); + } + @Override public long bytes() { return mBytes.get(); @@ -206,6 +227,56 @@ PageInfo evictInternal(CacheEvictor evictor) { return victimInfo; } + @Override + @GuardedBy("getLock().readLock()") + public Set getAllPagesByFileId(String fileId) { + Set pages = mPages.getByField(INDEX_FILE_ID, fileId); + return pages; + } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + class Usage implements CacheUsage { + + @Override + public long used() { + return bytes(); + } + + @Override + public long available() { + return capacity() - used(); + } + + @Override + public long capacity() { + return mDirs.stream().mapToLong(PageStoreDir::getCapacityBytes).sum(); + } + + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + if (partition instanceof FilePartition) { + String fileId = ((FilePartition) partition).getIdentifier(); + Set pages = mPages.getByField(INDEX_FILE_ID, fileId); + long used = pages.stream().mapToLong(PageInfo::getPageSize).sum(); + long capacity = capacity(); + long available = capacity - bytes(); + return Optional.of(new ImmutableCacheUsageView(used, available, capacity)); + } + if (partition instanceof DirPartition) { + int dirIndex = ((DirPartition) partition).getIdentifier(); + if (dirIndex < 0 || dirIndex >= mDirs.size()) { + return Optional.empty(); + } + return mDirs.get(dirIndex).getUsage(); + } + return Optional.empty(); + } + } + private static final class Metrics { // Note that only counter can be added here. // Both meter and timer need to be used inline diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java index fa78f002c55e..2ed8dae90a3f 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java @@ -17,14 +17,16 @@ import alluxio.client.file.CacheContext; import alluxio.client.file.FileInStream; import alluxio.client.file.URIStatus; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.ByteBufferTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.client.file.cache.context.CachePerThreadContext; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.PropertyKey; import alluxio.exception.AlluxioException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.ByteBufferTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.metrics.MultiDimensionalMetricsSystem; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -36,6 +38,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.NotThreadSafe; @@ -53,20 +56,20 @@ public class LocalCacheFileInStream extends FileInStream { /** Local store to store pages. */ private final CacheManager mCacheManager; + private final boolean mIsDora; private final boolean mQuotaEnabled; - /** Cache related context of this file. */ - private final CacheContext mCacheContext; /** File info, fetched from external FS. */ private final URIStatus mStatus; private final FileInStreamOpener mExternalFileInStreamOpener; private final int mBufferSize; + private final boolean mFallbackEnabled; private byte[] mBuffer = null; private long mBufferStartOffset; private long mBufferEndOffset; /** Stream reading from the external file system, opened once. */ - private FileInStream mExternalFileInStream; + private Optional mExternalFileInStream; /** Current position of the stream, relative to the start of the file. */ private long mPosition = 0; private boolean mClosed = false; @@ -99,20 +102,19 @@ public static void registerMetrics() { * @param fileOpener open file in the external file system if a cache miss occurs * @param cacheManager local cache manager * @param conf configuration + * @param externalFileInStream */ public LocalCacheFileInStream(URIStatus status, FileInStreamOpener fileOpener, - CacheManager cacheManager, AlluxioConfiguration conf) { + CacheManager cacheManager, AlluxioConfiguration conf, + Optional externalFileInStream) { mPageSize = conf.getBytes(PropertyKey.USER_CLIENT_CACHE_PAGE_SIZE); mExternalFileInStreamOpener = fileOpener; + mExternalFileInStream = externalFileInStream; mCacheManager = cacheManager; mStatus = status; + mIsDora = false; // Currently quota is only supported when it is set by external systems in status context mQuotaEnabled = conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED); - if (mQuotaEnabled && status.getCacheContext() != null) { - mCacheContext = status.getCacheContext(); - } else { - mCacheContext = CacheContext.defaults(); - } Metrics.registerGauges(); mBufferSize = (int) conf.getBytes(PropertyKey.USER_CLIENT_CACHE_IN_STREAM_BUFFER_SIZE); @@ -120,6 +122,7 @@ public LocalCacheFileInStream(URIStatus status, FileInStreamOpener fileOpener, if (mBufferSize > 0) { mBuffer = new byte[mBufferSize]; } + mFallbackEnabled = conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED); } @Override @@ -138,7 +141,7 @@ public int read(ByteBuffer buffer, int offset, int length) throws IOException { return totalBytesRead; } - private int bufferedRead(PageReadTargetBuffer targetBuffer, int length, + private int bufferedRead(ReadTargetBuffer targetBuffer, int length, ReadType readType, long position, Stopwatch stopwatch) throws IOException { if (mBuffer == null) { //buffer is disabled, read data from local cache directly. return localCachedRead(targetBuffer, length, readType, position, stopwatch); @@ -168,32 +171,31 @@ private int bufferedRead(PageReadTargetBuffer targetBuffer, int length, return dataReadFromBuffer; } - private int localCachedRead(PageReadTargetBuffer bytesBuffer, int length, + private int localCachedRead(ReadTargetBuffer bytesBuffer, int length, ReadType readType, long position, Stopwatch stopwatch) throws IOException { long currentPage = position / mPageSize; PageId pageId; CacheContext cacheContext = mStatus.getCacheContext(); - if (cacheContext != null && cacheContext.getCacheIdentifier() != null) { + if (cacheContext == null) { + cacheContext = CacheContext.defaults(); + } + if (cacheContext.getCacheIdentifier() != null) { pageId = new PageId(cacheContext.getCacheIdentifier(), currentPage); } else { - pageId = new PageId(Long.toString(mStatus.getFileId()), currentPage); + // In Dora, the fileId is generated by Worker or by local client, which maybe is not unique. + // So we use the ufs path hash as its fileId. + String fileId = Long.toString(mStatus.getFileId()); + pageId = new PageId(fileId, currentPage); } int currentPageOffset = (int) (position % mPageSize); int bytesLeftInPage = (int) (mPageSize - currentPageOffset); int bytesToReadInPage = Math.min(bytesLeftInPage, length); stopwatch.reset().start(); int bytesRead = - mCacheManager.get(pageId, currentPageOffset, bytesToReadInPage, bytesBuffer, mCacheContext); + mCacheManager.get(pageId, currentPageOffset, bytesToReadInPage, bytesBuffer, cacheContext); stopwatch.stop(); if (bytesRead > 0) { - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesRead); - if (cacheContext != null) { - cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), BYTE, - bytesRead); - cacheContext.incrementCounter( - MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), NANO, - stopwatch.elapsed(TimeUnit.NANOSECONDS)); - } + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).inc(); return bytesRead; } // on local cache miss, read a complete page from external storage. This will always make @@ -206,6 +208,7 @@ private int localCachedRead(PageReadTargetBuffer bytesBuffer, int length, // cache misses MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) .mark(bytesToReadInPage); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).inc(); if (cacheContext != null) { cacheContext.incrementCounter( MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getMetricName(), BYTE, @@ -215,13 +218,13 @@ private int localCachedRead(PageReadTargetBuffer bytesBuffer, int length, stopwatch.elapsed(TimeUnit.NANOSECONDS) ); } - mCacheManager.put(pageId, page, mCacheContext); + mCacheManager.put(pageId, page, cacheContext); } return bytesToReadInPage; } // TODO(binfan): take ByteBuffer once CacheManager takes ByteBuffer to avoid extra mem copy - private int readInternal(PageReadTargetBuffer targetBuffer, int offset, int length, + private int readInternal(ReadTargetBuffer targetBuffer, int offset, int length, ReadType readType, long position, boolean isPositionedRead) throws IOException { Preconditions.checkArgument(length >= 0, "length should be non-negative"); Preconditions.checkArgument(offset >= 0, "offset should be non-negative"); @@ -284,8 +287,27 @@ public long remaining() { @Override public int positionedRead(long pos, byte[] b, int off, int len) throws IOException { - return readInternal(new ByteArrayTargetBuffer(b, off), off, len, ReadType.READ_INTO_BYTE_ARRAY, - pos, true); + if (!CachePerThreadContext.get().getCacheEnabled()) { + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) + .mark(len); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).inc(); + len = getExternalFileInStream().positionedRead(pos, b, off, len); + MultiDimensionalMetricsSystem.EXTERNAL_DATA_READ.inc(len); + return len; + } + try { + return readInternal(new ByteArrayTargetBuffer(b, off), off, len, + ReadType.READ_INTO_BYTE_ARRAY, pos, true); + } catch (IOException | RuntimeException e) { + LOG.warn("Failed to read from Alluxio's page cache.", e); + if (mFallbackEnabled) { + MetricsSystem.counter(MetricKey.CLIENT_CACHE_POSITION_READ_FALLBACK.getName()).inc(); + len = getExternalFileInStream().positionedRead(pos, b, off, len); + MultiDimensionalMetricsSystem.EXTERNAL_DATA_READ.inc(len); + return len; + } + throw e; + } } @Override @@ -311,9 +333,7 @@ public void seek(long pos) { @Override public void unbuffer() { - if (mExternalFileInStream != null) { - mExternalFileInStream.unbuffer(); - } + mExternalFileInStream.ifPresent((stream) -> stream.unbuffer()); } /** @@ -331,19 +351,26 @@ private void checkIfClosed() { * @param position position to set the external stream to */ private FileInStream getExternalFileInStream(long position) throws IOException { + FileInStream externalFileInStream = getExternalFileInStream(); + long pageStart = position - (position % mPageSize); + if (externalFileInStream.getPos() != pageStart) { + externalFileInStream.seek(pageStart); + } + return externalFileInStream; + } + + private FileInStream getExternalFileInStream() throws IOException { try { - if (mExternalFileInStream == null) { - mExternalFileInStream = mExternalFileInStreamOpener.open(mStatus); - mCloser.register(mExternalFileInStream); + if (!mExternalFileInStream.isPresent()) { + FileInStream externalFileInStream = mExternalFileInStreamOpener.open(mStatus); + mExternalFileInStream = Optional.of(externalFileInStream); + mCloser.register(externalFileInStream); + return externalFileInStream; } + return mExternalFileInStream.get(); } catch (AlluxioException e) { throw new IOException(e); } - long pageStart = position - (position % mPageSize); - if (mExternalFileInStream.getPos() != pageStart) { - mExternalFileInStream.seek(pageStart); - } - return mExternalFileInStream; } /** @@ -386,6 +413,7 @@ private synchronized byte[] readExternalPage(long position, ReadType readType) } totalBytesRead += bytesRead; } + MultiDimensionalMetricsSystem.EXTERNAL_DATA_READ.inc(totalBytesRead); // Bytes read from external, may be larger than requests due to reading complete pages MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).mark(totalBytesRead); if (totalBytesRead != pageSize) { diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java index ed541972ce6b..f5673242dbef 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Optional; /** * A FileSystem implementation with a local cache. @@ -70,6 +71,7 @@ public FileInStream openFile(URIStatus status, OpenFilePOptions options) return mDelegatedFileSystem.openFile(status, options); } return new LocalCacheFileInStream(status, - uriStatus -> mDelegatedFileSystem.openFile(status, options), mCacheManager, mConf); + uriStatus -> mDelegatedFileSystem.openFile(status, options), mCacheManager, mConf, + Optional.empty()); } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java index f3edb5db43bc..4874820581cc 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java @@ -11,6 +11,8 @@ package alluxio.client.file.cache; +import static alluxio.client.file.CacheContext.StatsUnit.BYTE; +import static alluxio.client.file.CacheContext.StatsUnit.NANO; import static alluxio.client.file.cache.CacheManager.State.NOT_IN_USE; import static alluxio.client.file.cache.CacheManager.State.READ_ONLY; import static alluxio.client.file.cache.CacheManager.State.READ_WRITE; @@ -18,18 +20,23 @@ import static java.util.concurrent.TimeUnit.SECONDS; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreDir; import alluxio.client.quota.CacheQuota; import alluxio.client.quota.CacheScope; import alluxio.collections.ConcurrentHashSet; import alluxio.collections.Pair; +import alluxio.exception.FileDoesNotExistException; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.metrics.MultiDimensionalMetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.resource.LockResource; +import alluxio.util.ThreadFactoryUtils; import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; @@ -43,6 +50,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; @@ -54,6 +62,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; +import java.util.function.Supplier; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -61,12 +70,12 @@ /** * A class to manage & serve cached pages. This class coordinates various components to respond for * thread-safety and enforce cache replacement policies. - * + *

* The idea of creating a client-side cache is followed after "Improving In-Memory File System * Reading Performance by Fine-Grained User-Space Cache Mechanisms" by Gu et al, which illustrates * performance benefits for various read workloads. This class also introduces paging as a caching * unit. - * + *

* Lock hierarchy in this class: All operations must follow this order to operate on pages: *

    *
  1. Acquire corresponding page lock
    2. @@ -83,29 +92,38 @@ public class LocalCacheManager implements CacheManager { private static final int LOCK_SIZE = 1024; private final long mCacheSize; - /** A readwrite lock pool to guard individual pages based on striping. */ + /** + * A readwrite lock pool to guard individual pages based on striping. + */ private final ReadWriteLock[] mPageLocks = new ReentrantReadWriteLock[LOCK_SIZE]; private final List mPageStoreDirs; @GuardedBy("PageMetaStore.getLock()") private final PageMetaStore mPageMetaStore; - /** Executor service for execute the init tasks. */ + /** + * Executor service for execute the init tasks. + */ private final Optional mInitService; - /** Executor service for execute the async cache tasks. */ + /** + * Executor service for execute the async cache tasks. + */ private final Optional mAsyncCacheExecutor; /** Executor service for execute the cache ttl check tasks. */ private final Optional mTtlEnforcerExecutor; private final ConcurrentHashSet mPendingRequests; - /** State of this cache. */ + /** + * State of this cache. + */ private final AtomicReference mState = new AtomicReference<>(); private final CacheManagerOptions mOptions; + private final Optional> mPagePredicate; /** - * @param options the options of local cache manager + * @param options the options of local cache manager * @param pageMetaStore the metadata store for local cache * @return an instance of {@link LocalCacheManager} */ public static LocalCacheManager create(CacheManagerOptions options, - PageMetaStore pageMetaStore) + PageMetaStore pageMetaStore) throws IOException { LocalCacheManager manager = new LocalCacheManager(options, pageMetaStore); List pageStoreDirs = pageMetaStore.getStoreDirs(); @@ -124,7 +142,7 @@ public static LocalCacheManager create(CacheManagerOptions options, } /** - * @param options the options of local cache manager + * @param options the options of local cache manager * @param pageMetaStore the meta store manages the metadata */ @VisibleForTesting @@ -139,34 +157,98 @@ public static LocalCacheManager create(CacheManagerOptions options, mPendingRequests = new ConcurrentHashSet<>(); mAsyncCacheExecutor = options.isAsyncWriteEnabled() - ? Optional.of( - new ThreadPoolExecutor(mOptions.getAsyncWriteThreads(), - mOptions.getAsyncWriteThreads(), 60, TimeUnit.SECONDS, - new SynchronousQueue<>(), new ThreadPoolExecutor.CallerRunsPolicy())) + ? Optional.of(new ThreadPoolExecutor(mOptions.getAsyncWriteThreads(), + mOptions.getAsyncWriteThreads(), 60, TimeUnit.SECONDS, + new SynchronousQueue<>(), ThreadFactoryUtils.build( + "alluxio-async-cache-executor", true), + new ThreadPoolExecutor.CallerRunsPolicy())) : Optional.empty(); mInitService = - options.isAsyncRestoreEnabled() ? Optional.of(Executors.newSingleThreadExecutor()) : + options.isAsyncRestoreEnabled() ? Optional.of(Executors.newSingleThreadExecutor( + ThreadFactoryUtils.build("alluxio-init-service", true))) : Optional.empty(); if (options.isTtlEnabled()) { - mTtlEnforcerExecutor = Optional.of(newScheduledThreadPool(1)); + Predicate ttl = pageInfo -> { + try { + return System.currentTimeMillis() - pageInfo.getCreatedTimestamp() + >= options.getTtlThresholdSeconds() * 1000; + } catch (Exception ex) { + // In case of any exception, do not invalidate the cache + return false; + } + }; + mPagePredicate = Optional.of(ttl); + mTtlEnforcerExecutor = Optional.of(newScheduledThreadPool(1, + ThreadFactoryUtils.build("alluxio-ttl-executor", true))); mTtlEnforcerExecutor.get().scheduleAtFixedRate(() -> - LocalCacheManager.this.invalidate(pageInfo -> { - try { - return System.currentTimeMillis() - pageInfo.getCreatedTimestamp() - >= options.getTtlThresholdSeconds() * 1000; - } catch (Exception ex) { - // In case of any exception, do not invalidate the cache - return false; - } - }), 0, options.getTtlCheckIntervalSeconds(), SECONDS); + LocalCacheManager.this.invalidate(ttl), 0, options.getTtlCheckIntervalSeconds(), SECONDS); } else { mTtlEnforcerExecutor = Optional.empty(); + mPagePredicate = Optional.empty(); } Metrics.registerGauges(mCacheSize, mPageMetaStore); mState.set(READ_ONLY); Metrics.STATE.inc(); } + @Override + public Optional getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, CacheContext cacheContext) + throws PageNotFoundException { + Preconditions.checkArgument(pageOffset <= mOptions.getPageSize(), + "Read exceeds page boundary: offset=%s size=%s", + pageOffset, mOptions.getPageSize()); + LOG.debug("get({},pageOffset={}) enters", pageId, pageOffset); + if (mState.get() == NOT_IN_USE) { + Metrics.GET_NOT_READY_ERRORS.inc(); + Metrics.GET_ERRORS.inc(); + throw new PageNotFoundException(String.format("Page %s could not be found", pageId)); + } + ReadWriteLock pageLock = getPageLock(pageId); + long startTime = System.nanoTime(); + try (LockResource r = new LockResource(pageLock.readLock())) { + PageInfo pageInfo; + try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { + pageInfo = mPageMetaStore.getPageInfo(pageId); //check if page exists and refresh LRU items + } catch (PageNotFoundException e) { + LOG.debug("getDataChannel({},pageOffset={}) fails due to page not found in metastore", + pageId, pageOffset); + return Optional.empty(); + } + + try { + DataFileChannel dataFileChannel = pageInfo.getLocalCacheDir().getPageStore() + .getDataFileChannel(pageInfo.getPageId(), pageOffset, bytesToRead, + cacheContext.isTemporary()); + MultiDimensionalMetricsSystem.CACHED_DATA_READ.inc(bytesToRead); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).inc(); + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesToRead); + cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), BYTE, + bytesToRead); + LOG.debug("getDataChannel({},pageOffset={}) exits", pageId, pageOffset); + return Optional.of(dataFileChannel); + } catch (PageNotFoundException e) { + LOG.debug("getDataChannel({},pageOffset={}) fails due to page file not found", + pageId, pageOffset); + Metrics.GET_ERRORS.inc(); + Metrics.GET_STORE_READ_ERRORS.inc(); + // something is wrong to read this page, let's remove it from meta store + try (LockResource r2 = new LockResource(mPageMetaStore.getLock().writeLock())) { + mPageMetaStore.removePage(pageId); + return Optional.empty(); + } catch (PageNotFoundException ex) { + // best effort to remove this page from meta store and ignore the exception + Metrics.CLEANUP_GET_ERRORS.inc(); + return Optional.empty(); + } + } + } finally { + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), NANO, + System.nanoTime() - startTime); + } + } + /** * @param pageId page identifier * @return the page lock id @@ -221,9 +303,9 @@ enum PutResult { */ @Nullable private CacheScope checkScopeToEvict(int pageSize, - PageStoreDir pageStoreDir, - CacheScope scope, CacheQuota quota, - boolean forcedToEvict) { + PageStoreDir pageStoreDir, + CacheScope scope, CacheQuota quota, + boolean forcedToEvict) { if (mOptions.isQuotaEnabled()) { // Check quota usage for each scope for (CacheScope currentScope = scope; currentScope != null; @@ -325,9 +407,32 @@ private boolean putInternal(PageId pageId, ByteBuffer page, CacheContext cacheCo return false; } + @Override + public void commitFile(String fileId) { + // TODO(JiamingMai): we still need to commit the data (not only the page metadata) + // call commit method of PageStoreDir + try { + PageStoreDir dir = mPageMetaStore.getStoreDirOfFile(fileId); + dir.commit(fileId, fileId); + } catch (FileDoesNotExistException notExistException) { + LOG.error(notExistException.getMessage()); + } catch (IllegalStateException illegalStateException) { + // ignore the commit exception + LOG.error(illegalStateException.getMessage()); + } catch (IOException ioException) { + // ignore the commit exception + LOG.error(ioException.getMessage()); + } + } + private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheContext, - boolean forcedToEvict) { + boolean forcedToEvict) { LOG.debug("putInternal({},{} bytes) enters", pageId, page.remaining()); + if (pageId.getPageIndex() > 0 && page.remaining() == 0) { + LOG.error("cannot put an empty page except for the first page." + + "pageId={}, isTemporary={}", pageId, cacheContext.isTemporary()); + return PutResult.OTHER; + } PageInfo victimPageInfo = null; CacheScope scopeToEvict; ReadWriteLock pageLock = getPageLock(pageId); @@ -385,7 +490,7 @@ private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheC Pair pageLockPair = getPageLockPair(pageId, victimPageInfo.getPageId()); try (LockResource r1 = new LockResource(pageLockPair.getFirst().writeLock()); - LockResource r2 = new LockResource(pageLockPair.getSecond().writeLock())) { + LockResource r2 = new LockResource(pageLockPair.getSecond().writeLock())) { // Excise a two-phase commit to evict victim and add new page: // phase1: remove victim and add new page in metastore in a critical section protected by // metalock. Evictor will be updated inside metastore. @@ -413,6 +518,7 @@ private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheC try { pageStoreDir.getPageStore().delete(victim); // Bytes evicted from the cache + MultiDimensionalMetricsSystem.CACHED_EVICTED_DATA.inc(victimPageInfo.getPageSize()); MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_EVICTED.getName()) .mark(victimPageInfo.getPageSize()); // Errors when adding pages @@ -456,7 +562,7 @@ private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheC } private void addPageToMetaStore(PageId pageId, ByteBuffer page, CacheContext cacheContext, - PageStoreDir pageStoreDir) { + PageStoreDir pageStoreDir) { PageInfo pageInfo = new PageInfo(pageId, page.remaining(), cacheContext.getCacheScope(), pageStoreDir); if (cacheContext.isTemporary()) { @@ -477,8 +583,26 @@ private void undoAddPage(PageId pageId) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, - CacheContext cacheContext) { + public int get(PageId pageId, int pageOffset, ReadTargetBuffer buffer, + CacheContext cacheContext) { + ReadWriteLock pageLock = getPageLock(pageId); + long pageSize = -1L; + try (LockResource r = new LockResource(pageLock.readLock())) { + PageInfo pageInfo; + try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { + pageInfo = mPageMetaStore.getPageInfo(pageId); //check if page exists and refresh LRU items + } catch (PageNotFoundException e) { + LOG.debug("get({},pageOffset={}) fails due to page not found", pageId, pageOffset); + return 0; + } + pageSize = pageInfo.getPageSize(); + } + return get(pageId, pageOffset, (int) pageSize, buffer, cacheContext); + } + + @Override + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, + CacheContext cacheContext) { Preconditions.checkArgument(pageOffset <= mOptions.getPageSize(), "Read exceeds page boundary: offset=%s size=%s", pageOffset, mOptions.getPageSize()); Preconditions.checkArgument(bytesToRead <= buffer.remaining(), @@ -491,6 +615,7 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return -1; } ReadWriteLock pageLock = getPageLock(pageId); + long startTime = System.nanoTime(); try (LockResource r = new LockResource(pageLock.readLock())) { PageInfo pageInfo; try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { @@ -513,14 +638,61 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf } return -1; } + MultiDimensionalMetricsSystem.CACHED_DATA_READ.inc(bytesRead); + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesRead); + cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), BYTE, + bytesRead); LOG.debug("get({},pageOffset={}) exits", pageId, pageOffset); return bytesRead; + } finally { + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), NANO, + System.nanoTime() - startTime); } } @Override - public boolean delete(PageId pageId) { - LOG.debug("delete({}) enters", pageId); + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, + CacheContext cacheContext, Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).inc(); + return bytesRead; + } + // on local cache miss, read a complete page from external storage. This will always make + // progress or throw an exception + // Note that we cannot synchronize on the new page, as this will cause deadlock due to + // incompatible lock order within putAttempt + // Not synchronizing may cause the same page to be read multiple times from UFS by two or more + // concurrent requests, + // but this is acceptable, and is expected to be rare as long as the number of striping locks + // is big enough + long startTime = System.nanoTime(); + byte[] page = externalDataSupplier.get(); + long timeElapse = System.nanoTime() - startTime; + buffer.writeBytes(page, pageOffset, bytesToRead); + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) + .mark(bytesToRead); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).inc(); + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getMetricName(), BYTE, + bytesToRead); + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_PAGE_READ_EXTERNAL_TIME_NS.getMetricName(), NANO, + timeElapse); + put(pageId, page, cacheContext); + return bytesToRead; + } + + /** + * delete the specified page. + * + * @param pageId page identifier + * @param isTemporary whether is it temporary or not + * @return whether the page is deleted successfully or not + */ + public boolean delete(PageId pageId, boolean isTemporary) { if (mState.get() != READ_WRITE) { Metrics.DELETE_NOT_READY_ERRORS.inc(); Metrics.DELETE_ERRORS.inc(); @@ -531,15 +703,15 @@ public boolean delete(PageId pageId) { PageInfo pageInfo; try (LockResource r1 = new LockResource(mPageMetaStore.getLock().writeLock())) { try { - pageInfo = mPageMetaStore.removePage(pageId); + pageInfo = mPageMetaStore.removePage(pageId, isTemporary); } catch (PageNotFoundException e) { - LOG.error("Failed to delete page {} from metaStore ", pageId, e); + LOG.debug("Failed to delete page {} from metaStore ", pageId, e); Metrics.DELETE_NON_EXISTING_PAGE_ERRORS.inc(); Metrics.DELETE_ERRORS.inc(); return false; } } - boolean ok = deletePage(pageInfo); + boolean ok = deletePage(pageInfo, isTemporary); LOG.debug("delete({}) exits, success: {}", pageId, ok); if (!ok) { Metrics.DELETE_STORE_DELETE_ERRORS.inc(); @@ -549,6 +721,11 @@ public boolean delete(PageId pageId) { } } + @Override + public boolean delete(PageId pageId) { + return delete(pageId, false); + } + @Override public State state() { return mState.get(); @@ -563,8 +740,11 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac } if (appendAt > 0) { byte[] newPage = new byte[appendAt + page.length]; - get(pageId, 0, appendAt, new ByteArrayTargetBuffer(newPage, 0), cacheContext); - delete(pageId); + int readBytes = get(pageId, 0, appendAt, + new ByteArrayTargetBuffer(newPage, 0), cacheContext); + boolean success = delete(pageId, cacheContext.isTemporary()); + LOG.debug("delete pageId: {}, appendAt: {}, readBytes: {}, success: {}", + pageId, appendAt, readBytes, success); System.arraycopy(page, 0, newPage, appendAt, page.length); return put(pageId, newPage, cacheContext); } @@ -575,6 +755,7 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac * Restores a page store at the configured location, updating meta store accordingly. * If restore process fails, cleanup the location and create a new page store. * This method is synchronized to ensure only one thread can enter and operate. + * * @param pageStoreDirs */ private void restoreOrInit(List pageStoreDirs) throws IOException { @@ -600,15 +781,28 @@ private void restoreOrInit(List pageStoreDirs) throws IOException } private boolean restore(PageStoreDir pageStoreDir) { + long restoredPages = mPageMetaStore.numPages(); + long restoredBytes = mPageMetaStore.bytes(); + long discardPages = Metrics.PAGE_DISCARDED.getCount(); + long discardBytes = Metrics.BYTE_DISCARDED.getCount(); LOG.info("Restoring PageStoreDir ({})", pageStoreDir.getRootPath()); + if (!Files.exists(pageStoreDir.getRootPath())) { LOG.error("Failed to restore PageStore: Directory {} does not exist", pageStoreDir.getRootPath()); return false; } try { - pageStoreDir.scanPages(pageInfo -> { - addPageToDir(pageStoreDir, pageInfo.get()); + pageStoreDir.scanPages(optionalPageInfo -> { + if (optionalPageInfo.isPresent()) { + PageInfo pageInfo = optionalPageInfo.get(); + if (mPagePredicate.isPresent()) { + addPageBasedOnPredicate(pageStoreDir, pageInfo); + } + else { + addPageToDir(pageStoreDir, pageInfo); + } + } }); } catch (IOException | RuntimeException e) { LOG.error("Failed to restore PageStore", e); @@ -616,11 +810,40 @@ private boolean restore(PageStoreDir pageStoreDir) { } LOG.info("PageStore ({}) restored with {} pages ({} bytes), " + "discarded {} pages ({} bytes)", - pageStoreDir.getRootPath(), mPageMetaStore.numPages(), mPageMetaStore.bytes(), - Metrics.PAGE_DISCARDED.getCount(), Metrics.BYTE_DISCARDED); + pageStoreDir.getRootPath(), mPageMetaStore.numPages() - restoredPages, + mPageMetaStore.bytes() - restoredBytes, Metrics.PAGE_DISCARDED.getCount() - discardPages, + Metrics.BYTE_DISCARDED.getCount() - discardBytes); return true; } + private void addPageBasedOnPredicate(PageStoreDir pageStoreDir, PageInfo pageInfo) { + boolean tested = mPagePredicate.get().test(pageInfo); + if (!tested) { + addPageToDir(pageStoreDir, pageInfo); + MetricsSystem.histogram(MetricKey.CLIENT_CACHE_PAGES_AGES.getName()) + .update(System.currentTimeMillis() - pageInfo.getCreatedTimestamp()); + } + else { + // delete page directly and no metadata put into meta store + ReadWriteLock pageLock = getPageLock(pageInfo.getPageId()); + boolean isPageDeleted; + try (LockResource r = new LockResource(pageLock.writeLock())) { + isPageDeleted = deletePage(pageInfo, false); + } + if (isPageDeleted) { + MetricsSystem.meter(MetricKey.CLIENT_CACHE_PAGES_INVALIDATED.getName()).mark(); + } + else { + LOG.debug("Failed to delete old pages {} when restore", pageInfo.getPageId()); + Metrics.DELETE_STORE_DELETE_ERRORS.inc(); + Metrics.DELETE_ERRORS.inc(); + // still show the age of the old page to warn users + MetricsSystem.histogram(MetricKey.CLIENT_CACHE_PAGES_AGES.getName()) + .update(System.currentTimeMillis() - pageInfo.getCreatedTimestamp()); + } + } + } + private void addPageToDir(PageStoreDir pageStoreDir, PageInfo pageInfo) { PageId pageId = pageInfo.getPageId(); ReadWriteLock pageLock = getPageLock(pageId); @@ -661,16 +884,56 @@ public List getCachedPageIdsByFileId(String fileId, long fileLength) { return pageIds; } + @Override + public boolean hasPageUnsafe(PageId pageId) { + return mPageMetaStore.hasPage(pageId); + } + + @Override + public void deleteFile(String fileId) { + Set pages; + try (LockResource r = new LockResource(mPageMetaStore.getLock().readLock())) { + pages = mPageMetaStore.getAllPagesByFileId(fileId); + } + pages.forEach(page -> delete(page.getPageId())); + } + + @Override + public void deleteTempFile(String fileId) { + Set pages; + try (LockResource r = new LockResource(mPageMetaStore.getLock().readLock())) { + pages = mPageMetaStore.getAllPagesByFileId(fileId); + } + pages.forEach(page -> delete(page.getPageId(), true)); + } + @Override public void invalidate(Predicate predicate) { + if (mState.get() != READ_WRITE) { + Metrics.DELETE_NOT_READY_ERRORS.inc(); + Metrics.DELETE_ERRORS.inc(); + return; + } mPageStoreDirs.forEach(dir -> { try { - dir.scanPages(pageInfo -> { - if (pageInfo.isPresent() && predicate.test(pageInfo.get())) { - delete(pageInfo.get().getPageId()); + dir.scanPages(pageInfoOpt -> { + if (pageInfoOpt.isPresent()) { + PageInfo pageInfo = pageInfoOpt.get(); + boolean isPageDeleted = false; + if (predicate.test(pageInfo)) { + isPageDeleted = delete(pageInfo.getPageId()); + } + if (isPageDeleted) { + MetricsSystem.meter(MetricKey.CLIENT_CACHE_PAGES_INVALIDATED.getName()).mark(); + } + else { + MetricsSystem.histogram(MetricKey.CLIENT_CACHE_PAGES_AGES.getName()) + .update(System.currentTimeMillis() - pageInfo.getCreatedTimestamp()); + } } }); } catch (IOException e) { + LOG.error("IOException occurs in page scan", e); throw new RuntimeException(e); } }); @@ -678,7 +941,7 @@ public void invalidate(Predicate predicate) { @Override public void close() throws Exception { - for (PageStoreDir pageStoreDir: mPageStoreDirs) { + for (PageStoreDir pageStoreDir : mPageStoreDirs) { pageStoreDir.close(); } mPageMetaStore.reset(); @@ -694,18 +957,20 @@ public void close() throws Exception { * @param pageInfo page info * @return true if successful, false otherwise */ - private boolean deletePage(PageInfo pageInfo) { + private boolean deletePage(PageInfo pageInfo, boolean isTemporary) { try { - pageInfo.getLocalCacheDir().getPageStore().delete(pageInfo.getPageId()); + pageInfo.getLocalCacheDir().getPageStore().delete(pageInfo.getPageId(), isTemporary); } catch (IOException | PageNotFoundException e) { - LOG.error("Failed to delete page {} from pageStore", pageInfo.getPageId(), e); + LOG.error("Failed to delete page {} (isTemporary: {}) from pageStore.", + pageInfo.getPageId(), isTemporary, e); return false; } return true; } private int getPage(PageInfo pageInfo, int pageOffset, int bytesToRead, - PageReadTargetBuffer target, CacheContext cacheContext) { + ReadTargetBuffer target, CacheContext cacheContext) { + int originOffset = target.offset(); try { int ret = pageInfo.getLocalCacheDir().getPageStore() .get(pageInfo.getPageId(), pageOffset, bytesToRead, target, @@ -714,80 +979,168 @@ private int getPage(PageInfo pageInfo, int pageOffset, int bytesToRead, // data read from page store is inconsistent from the metastore LOG.error("Failed to read page {}: supposed to read {} bytes, {} bytes actually read", pageInfo.getPageId(), bytesToRead, ret); + target.offset(originOffset); //reset the offset + //best efforts to delete the corrupted file without acquire the write lock + deletePage(pageInfo, false); return -1; } + } catch (PageCorruptedException e) { + LOG.error("Data corrupted page {} from pageStore", pageInfo.getPageId(), e); + target.offset(originOffset); //reset the offset + //best efforts to delete the corrupted file without acquire the write lock + deletePage(pageInfo, false); + return -1; } catch (IOException | PageNotFoundException e) { LOG.debug("Failed to get existing page {} from pageStore", pageInfo.getPageId(), e); + target.offset(originOffset); //reset the offset return -1; } return bytesToRead; } + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + private final class Usage implements CacheUsage { + @Override + public long used() { + return mPageMetaStore.bytes(); + } + + @Override + public long available() { + return capacity() - used(); + } + + @Override + public long capacity() { + return mPageStoreDirs.stream().mapToLong(PageStoreDir::getCapacityBytes).sum(); + } + + @Override + public Optional partitionedBy(PartitionDescriptor partitionDescriptor) { + if (partitionDescriptor instanceof DirPartition) { + int dirIndex = ((DirPartition) partitionDescriptor).getIdentifier(); + if (dirIndex >= 0 && dirIndex < mPageStoreDirs.size()) { + return mPageStoreDirs.get(dirIndex).getUsage(); + } else { + return Optional.empty(); + } + } + return mPageMetaStore.getUsage() + .flatMap(usage -> usage.partitionedBy(partitionDescriptor)); + } + } + private static final class Metrics { // Note that only counter/guage can be added here. // Both meter and timer need to be used inline // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - /** Total number of bytes discarded when restoring the page store. */ + /** + * Total number of bytes discarded when restoring the page store. + */ private static final Counter BYTE_DISCARDED = MetricsSystem.counter(MetricKey.CLIENT_CACHE_BYTES_DISCARDED.getName()); - /** Errors when cleaning up a failed get operation. */ + /** + * Errors when cleaning up a failed get operation. + */ private static final Counter CLEANUP_GET_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_CLEANUP_GET_ERRORS.getName()); - /** Errors when cleaning up a failed put operation. */ + /** + * Errors when cleaning up a failed put operation. + */ private static final Counter CLEANUP_PUT_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_CLEANUP_PUT_ERRORS.getName()); - /** Errors when deleting pages. */ + /** + * Errors when deleting pages. + */ private static final Counter DELETE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_ERRORS.getName()); - /** Errors when deleting pages due to absence. */ + /** + * Errors when deleting pages due to absence. + */ private static final Counter DELETE_NON_EXISTING_PAGE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_NON_EXISTING_PAGE_ERRORS.getName()); - /** Errors when cache is not ready to delete pages. */ + /** + * Errors when cache is not ready to delete pages. + */ private static final Counter DELETE_NOT_READY_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_NOT_READY_ERRORS.getName()); - /** Errors when deleting pages due to failed delete in page stores. */ + /** + * Errors when deleting pages due to failed delete in page stores. + */ private static final Counter DELETE_STORE_DELETE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_FROM_STORE_ERRORS.getName()); - /** Errors when getting pages. */ + /** + * Errors when getting pages. + */ private static final Counter GET_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_ERRORS.getName()); - /** Errors when cache is not ready to get pages. */ + /** + * Errors when cache is not ready to get pages. + */ private static final Counter GET_NOT_READY_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_NOT_READY_ERRORS.getName()); - /** Errors when getting pages due to failed read from page stores. */ + /** + * Errors when getting pages due to failed read from page stores. + */ private static final Counter GET_STORE_READ_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_STORE_READ_ERRORS.getName()); - /** Total number of pages discarded when restoring the page store. */ + /** + * Total number of pages discarded when restoring the page store. + */ private static final Counter PAGE_DISCARDED = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PAGES_DISCARDED.getName()); - /** Errors when adding pages. */ + /** + * Errors when adding pages. + */ private static final Counter PUT_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ERRORS.getName()); - /** Errors when adding pages due to failed injection to async write queue. */ + /** + * Errors when adding pages due to failed injection to async write queue. + */ private static final Counter PUT_ASYNC_REJECTION_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ASYNC_REJECTION_ERRORS.getName()); - /** Errors when adding pages due to failed eviction. */ + /** + * Errors when adding pages due to failed eviction. + */ private static final Counter PUT_EVICTION_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_EVICTION_ERRORS.getName()); - /** Errors when adding pages due to benign racing eviction. */ + /** + * Errors when adding pages due to benign racing eviction. + */ private static final Counter PUT_BENIGN_RACING_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_BENIGN_RACING_ERRORS.getName()); - /** Errors when adding pages due to insufficient space made after eviction. */ + /** + * Errors when adding pages due to insufficient space made after eviction. + */ private static final Counter PUT_INSUFFICIENT_SPACE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_INSUFFICIENT_SPACE_ERRORS.getName()); - /** Errors when cache is not ready to add pages. */ + /** + * Errors when cache is not ready to add pages. + */ private static final Counter PUT_NOT_READY_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_NOT_READY_ERRORS.getName()); - /** Errors when adding pages due to failed deletes in page store. */ + /** + * Errors when adding pages due to failed deletes in page store. + */ private static final Counter PUT_STORE_DELETE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_DELETE_ERRORS.getName()); - /** Errors when adding pages due to failed writes to page store. */ + /** + * Errors when adding pages due to failed writes to page store. + */ private static final Counter PUT_STORE_WRITE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_WRITE_ERRORS.getName()); - /** Errors when adding pages due to failed writes but before reaching cache capacity. */ + /** + * Errors when adding pages due to failed writes but before reaching cache capacity. + */ private static final Counter PUT_STORE_WRITE_NO_SPACE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_WRITE_NO_SPACE_ERRORS.getName()); - /** State of the cache. */ + /** + * State of the cache. + */ private static final Counter STATE = MetricsSystem.counter(MetricKey.CLIENT_CACHE_STATE.getName()); diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java index 4438b496fd61..7e30bbe68abd 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java @@ -12,9 +12,11 @@ package alluxio.client.file.cache; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.codahale.metrics.Counter; import org.slf4j.Logger; @@ -22,6 +24,8 @@ import java.nio.ByteBuffer; import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; /** * A wrapper class of CacheManager without throwing unchecked exceptions. @@ -38,6 +42,15 @@ public NoExceptionCacheManager(CacheManager cacheManager) { mCacheManager = cacheManager; } + @Override + public void commitFile(String fileId) { + try { + mCacheManager.commitFile(fileId); + } catch (Exception e) { + LOG.error("Failed to commit file {}", fileId, e); + } + } + @Override public boolean put(PageId pageId, byte[] page) { try { @@ -60,6 +73,25 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } } + @Override + public int get(PageId pageId, int pageOffset, ReadTargetBuffer buffer, + CacheContext cacheContext) { + int originalOffset = buffer.offset(); + try { + int bytesRead = mCacheManager.get(pageId, pageOffset, buffer, cacheContext); + if (bytesRead == -1) { + buffer.offset(originalOffset); + } + return bytesRead; + } catch (Exception e) { + LOG.error("Failed to get page {}", pageId, e); + Metrics.GET_ERRORS.inc(); + //In case any error in cache manager, revert the offset change in the buffer + buffer.offset(originalOffset); + return -1; + } + } + @Override public int get(PageId pageId, int bytesToRead, byte[] buffer, int offsetInBuffer) { try { @@ -98,7 +130,7 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, CacheContext cacheContext) { try { return mCacheManager @@ -111,6 +143,21 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf } } + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + try { + return mCacheManager.getAndLoad(pageId, pageOffset, bytesToRead, + buffer, cacheContext, externalDataSupplier); + } catch (Exception e) { + LOG.error("Failed to get and load page {}, offset {} cacheContext {}", pageId, pageOffset, + cacheContext, e); + Metrics.GET_ERRORS.inc(); + return -1; + } + } + @Override public boolean delete(PageId pageId) { try { @@ -122,6 +169,23 @@ public boolean delete(PageId pageId) { } } + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + try { + return mCacheManager.getDataFileChannel(pageId, pageOffset, bytesToRead, cacheContext); + } catch (Exception e) { + if (e instanceof PageNotFoundException) { + // In cold read, this may be expected behavior + LOG.debug("Failed to getDataFileChannel of page {}", pageId, e); + } else { + LOG.error("Failed to getDataFileChannel of page {}", pageId, e); + } + Metrics.GET_ERRORS.inc(); + return Optional.empty(); + } + } + @Override public State state() { return mCacheManager.state(); @@ -146,6 +210,34 @@ public List getCachedPageIdsByFileId(String fileId, long fileLength) { return mCacheManager.getCachedPageIdsByFileId(fileId, fileLength); } + @Override + public boolean hasPageUnsafe(PageId pageId) { + return mCacheManager.hasPageUnsafe(pageId); + } + + @Override + public void deleteFile(String fileId) { + try { + mCacheManager.deleteFile(fileId); + } catch (Exception e) { + LOG.error("Failed to deleteFile for {}", fileId, e); + } + } + + @Override + public void deleteTempFile(String fileId) { + try { + mCacheManager.deleteTempFile(fileId); + } catch (Exception e) { + LOG.error("Failed to deleteFile for {}", fileId, e); + } + } + + @Override + public Optional getUsage() { + return mCacheManager.getUsage(); + } + private static final class Metrics { // Note that only counter/guage can be added here. // Both meter and timer need to be used inline diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java index 5bbaf90b5a9a..41ab24eaac0e 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java @@ -125,6 +125,7 @@ public String toString() { .add("PageId", mPageId) .add("PageSize", mPageSize) .add("Scope", mCacheScope) + .add("CreateTimeMs", mCreatedTimestamp) .toString(); } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java index 673955559fac..c918c64deb13 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java @@ -13,16 +13,18 @@ import alluxio.client.file.cache.store.PageStoreDir; import alluxio.client.quota.CacheScope; +import alluxio.exception.FileDoesNotExistException; import alluxio.exception.PageNotFoundException; import java.io.IOException; import java.util.List; +import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; /** * The metadata store for pages stored in cache. */ -public interface PageMetaStore { +public interface PageMetaStore extends CacheStatus { /** * @param options the options of cache @@ -71,6 +73,17 @@ static PageMetaStore create(CacheManagerOptions options) throws IOException { */ void commitFile(String fileId, String newFileId) throws PageNotFoundException; + /** + * Gets the store dir which the specified file is assigned to be cached in. + * Implementations should ensure that all pages + * of the same file are cached in the same directory, until all the pages of the file are evicted. + * + * @param fileId the file ID + * @return the store dir which caches the pages of the file + * @throws FileDoesNotExistException if the file is not being cached + */ + PageStoreDir getStoreDirOfFile(String fileId) throws FileDoesNotExistException; + /** * Gets the storage directories. * @@ -91,6 +104,15 @@ static PageMetaStore create(CacheManagerOptions options) throws IOException { */ PageInfo getPageInfo(PageId pageId) throws PageNotFoundException; + /** + * Removes a page. + * + * @param pageId page identifier + * @param isTemporary whether is it temporary page or not + * @return page info removed + */ + PageInfo removePage(PageId pageId, boolean isTemporary) throws PageNotFoundException; + /** * Removes a page. * @@ -114,6 +136,13 @@ static PageMetaStore create(CacheManagerOptions options) throws IOException { */ void reset(); + /** + * Gets all pages by the specified File ID. + * @param fileId the target file id + * @return a set of PageInfo's of this file, otherwise an empty set if not found + */ + Set getAllPagesByFileId(String fileId); + /** * @param pageStoreDir * @return a page to evict diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java index 660207295154..a73cd96a8939 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java @@ -11,15 +11,17 @@ package alluxio.client.file.cache; +import alluxio.Constants; import alluxio.client.file.cache.store.LocalPageStore; import alluxio.client.file.cache.store.MemoryPageStore; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; -import alluxio.client.file.cache.store.RocksPageStore; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; +import alluxio.util.logging.SamplingLogger; import com.codahale.metrics.Counter; import org.slf4j.Logger; @@ -34,6 +36,7 @@ */ public interface PageStore extends AutoCloseable { Logger LOG = LoggerFactory.getLogger(PageStore.class); + Logger SAMPLING_LOG = new SamplingLogger(LOG, Constants.SECOND_MS * 10); /** * Create an instance of PageStore. @@ -48,9 +51,6 @@ static PageStore create(PageStoreOptions options) { case LOCAL: pageStore = new LocalPageStore(options); break; - case ROCKS: - pageStore = RocksPageStore.open(options); - break; case MEM: pageStore = new MemoryPageStore((int) options.getPageSize()); break; @@ -127,7 +127,7 @@ void put(PageId pageId, * @throws IOException when the store fails to read this page * @throws PageNotFoundException when the page isn't found in the store */ - default int get(PageId pageId, PageReadTargetBuffer buffer) + default int get(PageId pageId, ReadTargetBuffer buffer) throws IOException, PageNotFoundException { return get(pageId, 0, (int) buffer.remaining(), buffer, false); } @@ -144,7 +144,7 @@ default int get(PageId pageId, PageReadTargetBuffer buffer) * @throws PageNotFoundException when the page isn't found in the store * @throws IllegalArgumentException when the page offset exceeds the page size */ - default int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer) + default int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer) throws IOException, PageNotFoundException { return get(pageId, pageOffset, bytesToRead, buffer, false); } @@ -162,10 +162,23 @@ default int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBu * @throws PageNotFoundException when the page isn't found in the store * @throws IllegalArgumentException when the page offset exceeds the page size */ - int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, boolean isTemporary) throws IOException, PageNotFoundException; + /** + * Deletes a temporary page from the store. + * + * @param pageId page identifier + * @param isTemporary whether is to delete the temporary page or not + * @throws IOException when the store fails to delete this page + * @throws PageNotFoundException when the page isn't found in the store + */ + default void delete(PageId pageId, boolean isTemporary) + throws IOException, PageNotFoundException { + delete(pageId); + } + /** * Deletes a page from the store. * @@ -173,7 +186,9 @@ int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buf * @throws IOException when the store fails to delete this page * @throws PageNotFoundException when the page isn't found in the store */ - void delete(PageId pageId) throws IOException, PageNotFoundException; + default void delete(PageId pageId) throws IOException, PageNotFoundException { + delete(pageId, false); + } /** * Commit a temporary file. @@ -200,6 +215,21 @@ default void abort(String fileId) throws IOException { throw new UnsupportedOperationException(); } + /** + * Get a {@link DataFileChannel} which wraps a {@link io.netty.channel.FileRegion}. + * @param pageId the page id + * @param pageOffset the offset inside the page + * @param bytesToRead the bytes to read + * @param isTemporary whether it is temporary or not + * @return an object of {@link DataFileChannel} + * @throws PageNotFoundException + */ + default DataFileChannel getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, boolean isTemporary) + throws PageNotFoundException { + throw new UnsupportedOperationException(); + } + /** * Metrics. */ diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java index 0ccb31257dfa..6e406dff8eed 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -120,4 +121,23 @@ public PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir) { CacheEvictor evictor = mCacheEvictors.computeIfAbsent(cacheScope, k -> mSupplier.get()); return evictInternal(evictor); } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + class Usage extends DefaultPageMetaStore.Usage { + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + if (partition instanceof ScopePartition) { + CacheScope scope = ((ScopePartition) partition).getIdentifier(); + long capacity = capacity(); + long used = bytes(scope); + long available = capacity - bytes(); // not capacity - used! + return Optional.of(new ImmutableCacheUsageView(used, available, capacity)); + } + return super.partitionedBy(partition); + } + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java index d3339355d5b7..553708c56e4e 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java @@ -11,12 +11,13 @@ package alluxio.client.file.cache; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.codahale.metrics.Counter; import com.google.common.base.Preconditions; @@ -88,7 +89,7 @@ public void put(PageId pageId, } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Callable callable = () -> mPageStore.get(pageId, pageOffset, bytesToRead, target, isTemporary); @@ -136,6 +137,12 @@ public void delete(PageId pageId) throws IOException, PageNotFoundException { } } + @Override + public DataFileChannel getDataFileChannel(PageId pageId, int pageOffset, int bytesToRead, + boolean isTemporary) throws PageNotFoundException { + return mPageStore.getDataFileChannel(pageId, pageOffset, bytesToRead, isTemporary); + } + @Override public void close() throws Exception { mExecutorService.shutdown(); diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java b/core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java new file mode 100644 index 000000000000..12b81ca4ad17 --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java @@ -0,0 +1,43 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache.context; + +/** + * A per-thread cache context for local cache. + */ +public class CachePerThreadContext { + private static ThreadLocal sContext = + ThreadLocal.withInitial(() -> new CachePerThreadContext()); + + private boolean mCacheEnabled = true; + + /** + * @return per-thread cache context + */ + public static CachePerThreadContext get() { + return sContext.get(); + } + + /** + * @param isCacheEnabled + */ + public void setCacheEnabled(boolean isCacheEnabled) { + mCacheEnabled = isCacheEnabled; + } + + /** + * @return if cache is enabled + */ + public boolean getCacheEnabled() { + return mCacheEnabled; + } +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java index 446c9e5e5a95..b972d944a129 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java @@ -15,13 +15,18 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; +import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; @@ -65,38 +70,44 @@ public void put(PageId pageId, boolean isTemporary) throws ResourceExhaustedException, IOException { Path pagePath = getPagePath(pageId, isTemporary); try { + LOG.debug("Put page: {}, page's position: {}, page's limit: {}, page's capacity: {}", + pageId, page.position(), page.limit(), page.capacity()); if (!Files.exists(pagePath)) { Path parent = Preconditions.checkNotNull(pagePath.getParent(), "parent of cache file should not be null"); Files.createDirectories(parent); - Files.createFile(pagePath); } // extra try to ensure output stream is closed try (FileOutputStream fos = new FileOutputStream(pagePath.toFile(), false)) { fos.getChannel().write(page); } - } catch (Exception e) { + } catch (Throwable t) { Files.deleteIfExists(pagePath); - if (e.getMessage() != null && e.getMessage().contains(ERROR_NO_SPACE_LEFT)) { + if (t.getMessage() != null && t.getMessage().contains(ERROR_NO_SPACE_LEFT)) { throw new ResourceExhaustedException( - String.format("%s is full, configured with %d bytes", mRoot, mCapacity), e); + String.format("%s is full, configured with %d bytes", mRoot, mCapacity), t); } - throw new IOException("Failed to write file " + pagePath + " for page " + pageId, e); + throw new IOException("Failed to write file " + pagePath + " for page " + pageId, t); } } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); - Path pagePath = getPagePath(pageId, isTemporary); - if (!Files.exists(pagePath)) { - throw new PageNotFoundException(pagePath.toString()); + Preconditions.checkArgument(bytesToRead >= 0, "bytes to read should be non-negative"); + if (target.remaining() == 0 || bytesToRead == 0) { + return 0; } - long pageLength = pagePath.toFile().length(); - Preconditions.checkArgument(pageOffset <= pageLength, "page offset %s exceeded page size %s", - pageOffset, pageLength); + Path pagePath = getPagePath(pageId, isTemporary); try (RandomAccessFile localFile = new RandomAccessFile(pagePath.toString(), "r")) { + long pageLength = localFile.length(); + if (pageOffset + bytesToRead > pageLength) { + throw new PageCorruptedException(String.format( + "The page %s (%s) probably has been corrupted, " + + "page-offset %s, bytes to read %s, page file length %s", + pageId, pagePath, pageOffset, bytesToRead, pageLength)); + } int bytesSkipped = localFile.skipBytes(pageOffset); if (pageOffset != bytesSkipped) { throw new IOException( @@ -104,8 +115,7 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf pageId, pagePath, pageOffset, bytesSkipped)); } int bytesRead = 0; - int bytesLeft = (int) Math.min(pageLength - pageOffset, target.remaining()); - bytesLeft = Math.min(bytesLeft, bytesToRead); + int bytesLeft = Math.min((int) target.remaining(), bytesToRead); while (bytesLeft > 0) { int bytes = target.readFromFile(localFile, bytesLeft); if (bytes <= 0) { @@ -114,13 +124,27 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf bytesRead += bytes; bytesLeft -= bytes; } + if (bytesRead == 0) { + SAMPLING_LOG.warn("Read 0 bytes from page {}, the page is probably empty", pageId); + // no bytes have been read at all, but the requested length > 0 + // this means the file is empty + return -1; + } return bytesRead; + } catch (FileNotFoundException e) { + throw new PageNotFoundException(pagePath.toString()); } } - @Override - public void delete(PageId pageId) throws IOException, PageNotFoundException { - Path pagePath = getPagePath(pageId, false); + /** + * + * @param pageId page identifier + * @param isTemporary whether is to delete a temporary page or not + * @throws IOException + * @throws PageNotFoundException + */ + public void delete(PageId pageId, boolean isTemporary) throws IOException, PageNotFoundException { + Path pagePath = getPagePath(pageId, isTemporary); if (!Files.exists(pagePath)) { throw new PageNotFoundException(pagePath.toString()); } @@ -189,6 +213,44 @@ public Path getPagePath(PageId pageId, boolean isTemporary) { return filePath.resolve(Long.toString(pageId.getPageIndex())); } + @Override + public DataFileChannel getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, boolean isTemporary) + throws PageNotFoundException { + Preconditions.checkArgument(pageOffset >= 0, + "page offset should be non-negative"); + Preconditions.checkArgument(!isTemporary, + "cannot acquire a data file channel to a temporary page"); + Path pagePath = getPagePath(pageId, isTemporary); + File pageFile = pagePath.toFile(); + if (!pageFile.exists()) { + throw new PageNotFoundException(pagePath.toString()); + } + + long fileLength = pageFile.length(); + if (fileLength == 0 && pageId.getPageIndex() > 0) { + // pages other than the first page should always be non-empty + // remove this malformed page + SAMPLING_LOG.warn("Length of page {} is 0, removing this malformed page", pageId); + try { + Files.deleteIfExists(pagePath); + } catch (IOException ignored) { + // do nothing + } + throw new PageNotFoundException(pagePath.toString()); + } + if (fileLength < pageOffset) { + throw new IllegalArgumentException( + String.format("offset %s exceeds length of page %s", pageOffset, fileLength)); + } + if (pageOffset + bytesToRead > fileLength) { + bytesToRead = (int) (fileLength - (long) pageOffset); + } + + DataFileChannel dataFileChannel = new DataFileChannel(pageFile, pageOffset, bytesToRead); + return dataFileChannel; + } + @Override public void close() { // no-op diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java index 3d13ba544313..c04d4e8da346 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java @@ -13,6 +13,7 @@ import static alluxio.client.file.cache.store.PageStoreDir.getFileBucket; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; @@ -100,7 +101,7 @@ public void scanPages(Consumer> pageInfoConsumer) throws IOEx /** * @param path path of a file - * @return the corresponding page info for the file otherwise null + * @return the corresponding page info for the file otherwise empty */ private Optional getPageInfo(Path path) { Optional pageId = getPageId(path); @@ -113,22 +114,47 @@ private Optional getPageInfo(Path path) { createdTime = creationTime.toMillis(); } catch (IOException e) { LOG.error("Failed to get file size for " + path, e); + deleteUnrecognizedPage(path); return Optional.empty(); } return Optional.of(new PageInfo(pageId.get(), pageSize, CacheScope.GLOBAL, this, createdTime)); } + deleteUnrecognizedPage(path); return Optional.empty(); } + /** + * Deletes an unrecognized page file due to various reasons. + * @param path the file path of a page file + */ + private void deleteUnrecognizedPage(Path path) { + Preconditions.checkState(path.startsWith(getRootPath()), + String.format("%s is not inside the cache dir (%s)!", path, getRootPath())); + try { + Files.delete(path); + } catch (IOException e) { + // ignore. + } + } + /** * @param path path of a file - * @return the corresponding page id, or null if the file name does not match the pattern + * @return the corresponding page id, or empty if the file name does not match the pattern */ private Optional getPageId(Path path) { Matcher matcher = mPagePattern.matcher(path.toString()); if (!matcher.matches()) { - LOG.error("Unrecognized page file " + path); + // @TODO(hua) define the mPagePattern and TEMP page Pattern as static class member to save + // CPU time and memory footprint. + if (Pattern.matches(String.format("%s/%d/%s/([^/]+)/(\\d+)", + Pattern.quote(mPageStoreOptions.getRootDir().toString()), + mPageStoreOptions.getPageSize(), LocalPageStore.TEMP_DIR), path.toString())) { + LOG.info("TEMP page file " + path + " is going to be deleted."); + } else { + LOG.error("Unrecognized page file " + path + "is going to be deleted."); + } + deleteUnrecognizedPage(path); return Optional.empty(); } try { @@ -136,6 +162,7 @@ private Optional getPageId(Path path) { String fileId = Preconditions.checkNotNull(matcher.group(2)); if (!fileBucket.equals(getFileBucket(mFileBuckets, fileId))) { LOG.error("Bucket number mismatch " + path); + deleteUnrecognizedPage(path); return Optional.empty(); } String fileName = Preconditions.checkNotNull(matcher.group(3)); @@ -143,7 +170,13 @@ private Optional getPageId(Path path) { return Optional.of(new PageId(fileId, pageIndex)); } catch (NumberFormatException e) { LOG.error("Illegal numbers in path " + path); + deleteUnrecognizedPage(path); return Optional.empty(); } } + + @Override + public Optional getUsage() { + return Optional.of(new QuotaManagedPageStoreDir.Usage()); + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java index 81dc14829c76..7fde058b6c27 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java @@ -13,11 +13,14 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.LinkedList; @@ -57,7 +60,7 @@ public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOEx } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Preconditions.checkArgument(target != null, "buffer is null"); Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); @@ -66,8 +69,12 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf throw new PageNotFoundException(pageId.getFileId() + "_" + pageId.getPageIndex()); } MemPage page = mPageStoreMap.get(pageKey); - Preconditions.checkArgument(pageOffset <= page.getPageLength(), - "page offset %s exceeded page size %s", pageOffset, page.getPageLength()); + if (pageOffset + bytesToRead > page.getPageLength()) { + throw new PageCorruptedException(String.format( + "The page %s probably has been corrupted, " + + "page-offset %s, bytes to read %s, page file length %s", + pageId, pageOffset, bytesToRead, page.getPageLength())); + } int bytesLeft = (int) Math.min(page.getPageLength() - pageOffset, target.remaining()); bytesLeft = Math.min(bytesLeft, bytesToRead); target.writeBytes(page.getPage(), pageOffset, bytesLeft); @@ -84,6 +91,11 @@ public void delete(PageId pageId) throws IOException, PageNotFoundException { mPageStoreMap.remove(pageKey); } + @Override + public void commit(String fileId, String newFileId) throws IOException { + // noop because the pages are all in memory, there is no underlying storage to commit to + } + /** * @param pageId page Id * @return the key to this page @@ -99,6 +111,7 @@ public PageId getKeyFromPageId(PageId pageId) { public void close() { mPageStoreMap.clear(); mPageStoreMap = null; + mPagePool.close(); } /** @@ -130,7 +143,7 @@ public void setPageLength(int pageLength) { } } - private static class PagePool { + private static class PagePool implements Closeable { private final int mPageSize; private final LinkedList mPool = new LinkedList<>(); @@ -154,5 +167,10 @@ public void release(MemPage page) { mPool.push(page); } } + + @Override + public void close() { + mPool.clear(); + } } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java index 4a26dae5ee67..ff987931ba5c 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java @@ -13,6 +13,7 @@ import static java.util.Objects.requireNonNull; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; import alluxio.client.file.cache.evictor.CacheEvictor; @@ -57,4 +58,9 @@ public void reset() { public void scanPages(Consumer> pageInfoConsumer) { //do nothing } + + @Override + public Optional getUsage() { + return Optional.of(new QuotaManagedPageStoreDir.Usage()); + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java index 0a3378b18588..992b7570a171 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java @@ -12,6 +12,7 @@ package alluxio.client.file.cache.store; import alluxio.client.file.cache.CacheManagerOptions; +import alluxio.client.file.cache.CacheStatus; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; import alluxio.client.file.cache.evictor.CacheEvictor; @@ -33,7 +34,7 @@ /** * Directory of page store. */ -public interface PageStoreDir { +public interface PageStoreDir extends CacheStatus { Logger LOG = LoggerFactory.getLogger(PageStoreDir.class); /** @@ -66,12 +67,6 @@ static PageStoreDir createPageStoreDir(CacheEvictorOptions cacheEvictorOptions, PageStore.create(pageStoreOptions), CacheEvictor.create(cacheEvictorOptions) ); - case ROCKS: - return new RocksPageStoreDir( - pageStoreOptions, - PageStore.create(pageStoreOptions), - CacheEvictor.create(cacheEvictorOptions) - ); case MEM: return new MemoryPageStoreDir( pageStoreOptions, @@ -168,6 +163,11 @@ static void clear(Path rootPath) throws IOException { */ boolean reserve(long bytes); + /** + * @param bytes + */ + void deleteTempPage(PageInfo bytes); + /** * @param bytes * @return the bytes used after release diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java index 4de49a2aff24..640a81a12017 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java @@ -13,14 +13,20 @@ import static com.google.common.base.Preconditions.checkState; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.evictor.CacheEvictor; import alluxio.resource.LockResource; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.concurrent.GuardedBy; @@ -35,6 +41,8 @@ abstract class QuotaManagedPageStoreDir implements PageStoreDir { @GuardedBy("mTempFileIdSetLock") private final Set mTempFileIdSet = new HashSet<>(); + private final Map> mTempFileToPageInfoListMap = new ConcurrentHashMap<>(); + private final Path mRootPath; private final long mCapacityBytes; private final AtomicLong mBytesUsed = new AtomicLong(0); @@ -73,9 +81,12 @@ public void putPage(PageInfo pageInfo) { @Override public void putTempPage(PageInfo pageInfo) { + mTempFileToPageInfoListMap.computeIfAbsent(pageInfo.getPageId().getFileId(), + tempFileId -> new ArrayList<>()).add(pageInfo); try (LockResource lock = new LockResource(mTempFileIdSetLock.readLock())) { mTempFileIdSet.add(pageInfo.getPageId().getFileId()); } + mBytesUsed.addAndGet(pageInfo.getPageSize()); } @Override @@ -84,6 +95,25 @@ public long deletePage(PageInfo pageInfo) { return mBytesUsed.addAndGet(-pageInfo.getPageSize()); } + @Override + public void deleteTempPage(PageInfo pageInfo) { + String fileId = pageInfo.getPageId().getFileId(); + if (mTempFileToPageInfoListMap.containsKey(fileId)) { + List pageInfoList = + mTempFileToPageInfoListMap.get(fileId); + if (pageInfoList != null && pageInfoList.contains(pageInfo)) { + pageInfoList.remove(pageInfo); + if (pageInfoList.isEmpty()) { + mTempFileToPageInfoListMap.remove(fileId); + try (LockResource lock = new LockResource(mTempFileIdSetLock.readLock())) { + mTempFileIdSet.remove(fileId); + } + } + } + } + mBytesUsed.addAndGet(-pageInfo.getPageSize()); + } + @Override public boolean putTempFile(String fileId) { try (LockResource lock = new LockResource(mTempFileIdSetLock.writeLock())) { @@ -142,10 +172,17 @@ public void commit(String fileId, String newFileId) throws IOException { try (LockResource tempFileIdSetlock = new LockResource(mTempFileIdSetLock.writeLock()); LockResource fileIdSetlock = new LockResource(mFileIdSetLock.writeLock())) { checkState(mTempFileIdSet.contains(fileId), "temp file does not exist " + fileId); - checkState(!mFileIdSet.contains(newFileId), "file already committed " + newFileId); + // We need a new interface for {@PageStore} interface to remove all pages of a cached file, + // and remove the fileId from this mFileIdSet. See {@DoraWorker#invalidateCachedFile}. + // Currently, invalidated file is still in this map. + //checkState(!mFileIdSet.contains(newFileId), "file already committed " + newFileId); getPageStore().commit(fileId, newFileId); mTempFileIdSet.remove(fileId); mFileIdSet.add(newFileId); + + mTempFileToPageInfoListMap.get(fileId) + .forEach(pageInfo -> mEvictor.updateOnPut(pageInfo.getPageId())); + mTempFileToPageInfoListMap.remove(fileId); } } @@ -157,4 +194,39 @@ public void abort(String fileId) throws IOException { mTempFileIdSet.remove(fileId); } } + + /** + * Generic implementation of cache usage stats. + * Subclasses may need to override the individual cache stat to reflect their own logic + * of usage accounting. + */ + class Usage implements CacheUsage { + @Override + public long used() { + return getCachedBytes(); + } + + @Override + public long available() { + // TODO(bowen): take reserved bytes into account + return getCapacityBytes() - getCachedBytes(); + } + + @Override + public long capacity() { + return getCapacityBytes(); + } + + /** + * This generic implementation assumes the directory does not support finer-grained + * stats partitioning. + * + * @param partition how to partition the cache + * @return always empty + */ + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java index 4526340d668f..1fb83c5394f6 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java @@ -15,6 +15,7 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import alluxio.proto.client.Cache; import com.google.common.base.Preconditions; @@ -177,7 +178,7 @@ public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOEx } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); try { diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java index 1f218ca56b44..f3ee48ea9fa6 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java @@ -13,6 +13,7 @@ import static com.google.common.base.Preconditions.checkState; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; @@ -83,4 +84,9 @@ private Optional parsePageInfo(RocksIterator iter) { } return Optional.empty(); } + + @Override + public Optional getUsage() { + return Optional.of(new QuotaManagedPageStoreDir.Usage()); + } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java b/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java index 1df3898b1054..fbebff6edb9b 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java +++ b/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java @@ -70,7 +70,7 @@ public long getPos() throws IOException { @Override public int positionedRead(long position, byte[] buffer, int offset, int length) throws IOException { - throw new UnsupportedOperationException("positionedRead not implemented for mock FileInStream"); + return length; } @Override diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java index 5888ca44556d..2a4fd57d8dfd 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java @@ -17,10 +17,11 @@ import alluxio.Constants; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.util.io.BufferUtils; import org.junit.Before; @@ -32,6 +33,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.Optional; +import java.util.function.Supplier; /** * Tests for the {@link LocalCacheManager} class. @@ -228,7 +231,7 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, CacheContext cacheContext) { if (!mCache.containsKey(pageId)) { return 0; @@ -241,6 +244,29 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return bytesToRead; } + @Override + public void commitFile(String fileId) { + throw new UnsupportedOperationException("commitFile method is unsupported. "); + } + + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + return bytesRead; + } + byte[] page = externalDataSupplier.get(); + if (page.length == 0) { + return 0; + } + buffer.writeBytes(page, pageOffset, bytesToRead); + put(pageId, page, cacheContext); + return bytesToRead; + } + @Override public boolean delete(PageId pageId) { if (mCache.containsKey(pageId)) { @@ -261,6 +287,27 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac } @Override - public void close() throws Exception {} + public void deleteFile(String fileId) { + // no-op + } + + @Override + public void deleteTempFile(String fileId) { + // no-op + } + + @Override + public Optional getUsage() { + return Optional.empty(); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + return Optional.empty(); + } + + @Override + public void close() {} } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java b/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java index fde5fafbc3c2..9917f03c82f2 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java @@ -12,9 +12,9 @@ package alluxio.client.file.cache; import alluxio.client.file.cache.store.LocalPageStore; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import java.io.IOException; import java.nio.ByteBuffer; @@ -44,7 +44,7 @@ public void delete(PageId pageId) throws IOException, PageNotFoundException { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { checkStopHanging(); diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java index ab17f2a98a7d..910225b940c8 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java @@ -11,6 +11,8 @@ package alluxio.client.file.cache; +import static org.junit.Assert.fail; + import alluxio.AlluxioURI; import alluxio.Constants; import alluxio.client.file.CacheContext; @@ -20,7 +22,7 @@ import alluxio.client.file.ListStatusPartialResult; import alluxio.client.file.MockFileInStream; import alluxio.client.file.URIStatus; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.client.file.cache.context.CachePerThreadContext; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; @@ -32,6 +34,7 @@ import alluxio.exception.FileIncompleteException; import alluxio.exception.InvalidPathException; import alluxio.exception.OpenDirectoryException; +import alluxio.file.ReadTargetBuffer; import alluxio.grpc.CancelSyncMetadataPResponse; import alluxio.grpc.CheckAccessPOptions; import alluxio.grpc.CreateDirectoryPOptions; @@ -59,6 +62,7 @@ import alluxio.job.JobRequest; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.security.authorization.AclEntry; import alluxio.util.io.BufferUtils; import alluxio.util.io.PathUtils; @@ -95,6 +99,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -123,6 +128,7 @@ public void before() { MetricsSystem.clearAllMetrics(); sConf.set(PropertyKey.USER_CLIENT_CACHE_PAGE_SIZE, mPageSize); sConf.set(PropertyKey.USER_CLIENT_CACHE_IN_STREAM_BUFFER_SIZE, mBufferSize); + CachePerThreadContext.get().setCacheEnabled(true); } @Test @@ -226,7 +232,8 @@ public void readPartialPageThroughReadByteBufferMethod() throws Exception { stream.seek(offset); Assert.assertEquals(partialReadSize, stream.read(cacheMissBuffer)); Assert.assertArrayEquals( - Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheMissBuffer.array()); + Arrays.copyOfRange(testData, offset, offset + partialReadSize), + cacheMissBuffer.array()); Assert.assertEquals(0, manager.mPagesServed); Assert.assertEquals(1, manager.mPagesCached); @@ -234,8 +241,8 @@ public void readPartialPageThroughReadByteBufferMethod() throws Exception { ByteBuffer cacheHitBuffer = ByteBuffer.wrap(new byte[partialReadSize]); stream.seek(offset); Assert.assertEquals(partialReadSize, stream.read(cacheHitBuffer)); - Assert.assertArrayEquals( - Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheHitBuffer.array()); + Assert.assertArrayEquals(Arrays.copyOfRange(testData, offset, offset + partialReadSize), + cacheHitBuffer.array()); Assert.assertEquals(1, manager.mPagesServed); } @@ -363,6 +370,10 @@ public void positionedReadPartialPage() throws Exception { Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheMiss); Assert.assertEquals(0, manager.mPagesServed); Assert.assertEquals(1, manager.mPagesCached); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); // cache hit byte[] cacheHit = new byte[partialReadSize]; @@ -371,6 +382,10 @@ public void positionedReadPartialPage() throws Exception { Assert.assertArrayEquals( Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheHit); Assert.assertEquals(1, manager.mPagesServed); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); } @Test @@ -382,7 +397,8 @@ public void positionReadOversizedBuffer() throws Exception { // cache miss byte[] cacheMiss = new byte[fileSize * 2]; - Assert.assertEquals(fileSize - 1, stream.positionedRead(1, cacheMiss, 2, fileSize * 2)); + Assert.assertEquals(fileSize - 1, + stream.positionedRead(1, cacheMiss, 2, fileSize * 2)); Assert.assertArrayEquals( Arrays.copyOfRange(testData, 1, fileSize - 1), Arrays.copyOfRange(cacheMiss, 2, fileSize)); @@ -391,7 +407,8 @@ public void positionReadOversizedBuffer() throws Exception { // cache hit byte[] cacheHit = new byte[fileSize * 2]; - Assert.assertEquals(fileSize - 1, stream.positionedRead(1, cacheHit, 2, fileSize * 2)); + Assert.assertEquals(fileSize - 1, + stream.positionedRead(1, cacheHit, 2, fileSize * 2)); Assert.assertArrayEquals( Arrays.copyOfRange(testData, 1, fileSize - 1), Arrays.copyOfRange(cacheHit, 2, fileSize)); @@ -415,15 +432,21 @@ public void readPagesMetrics() throws Exception { MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()).getCount()); Assert.assertEquals(fileSize, MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).getCount()); + Assert.assertEquals(5, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); // cache hit stream.read(); - Assert.assertEquals(1, - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).getCount()); Assert.assertEquals(readSize, MetricsSystem.meter( MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()).getCount()); Assert.assertEquals(fileSize, MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).getCount()); + Assert.assertEquals(5, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); } @Test @@ -438,7 +461,8 @@ public void externalStoreMultiRead() throws Exception { ByteArrayFileSystem fs = new MultiReadByteArrayFileSystem(files); LocalCacheFileInStream stream = new LocalCacheFileInStream(fs.getStatus(testFilename), - (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf); + (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf, + Optional.empty()); // cache miss byte[] cacheMiss = new byte[fileSize]; @@ -467,7 +491,8 @@ public void externalStoreMultiReadThroughReadByteBufferMethod() throws Exception ByteArrayFileSystem fs = new MultiReadByteArrayFileSystem(files); LocalCacheFileInStream stream = new LocalCacheFileInStream(fs.getStatus(testFilename), - (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf); + (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf, + Optional.empty()); // cache miss ByteBuffer cacheMissBuf = ByteBuffer.wrap(new byte[fileSize]); @@ -515,7 +540,7 @@ public void cacheMetricCacheHitReadTime() throws Exception { LocalCacheFileInStream stream = new LocalCacheFileInStream(fs.getStatus(testFileName), (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, - sConf) { + sConf, Optional.empty()) { @Override protected Stopwatch createUnstartedStopwatch() { return Stopwatch.createUnstarted(timeSource); @@ -523,11 +548,8 @@ protected Stopwatch createUnstartedStopwatch() { }; Assert.assertArrayEquals(testData, ByteStreams.toByteArray(stream)); - long timeReadCache = recordedMetrics.get( - MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName()); long timeReadExternal = recordedMetrics.get( MetricKey.CLIENT_CACHE_PAGE_READ_EXTERNAL_TIME_NS.getMetricName()); - Assert.assertEquals(timeSource.get(StepTicker.Type.CACHE_HIT), timeReadCache); Assert.assertEquals(timeSource.get(StepTicker.Type.CACHE_MISS), timeReadExternal); } @@ -563,6 +585,88 @@ public void testUnbuffer() throws Exception { Assert.assertEquals(1, manager.mPagesServed); } + @Test + public void testPageDataFileCorrupted() throws Exception + { + int pages = 10; + int fileSize = mPageSize * pages; + byte[] testData = BufferUtils.getIncreasingByteArray(fileSize); + ByteArrayCacheManager manager = new ByteArrayCacheManager(); + //by default local cache fallback is not enabled, the read should fail for any error + LocalCacheFileInStream streamWithOutFallback = setupWithSingleFile(testData, manager); + + sConf.set(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED, true); + LocalCacheFileInStream streamWithFallback = setupWithSingleFile(testData, manager); + Assert.assertEquals(100, streamWithFallback.positionedRead(0, new byte[10], 100, 100)); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_POSITION_READ_FALLBACK.getName()).getCount()); + } + + @Test + public void testPositionReadFallBack() throws Exception + { + int pages = 10; + int fileSize = mPageSize * pages; + byte[] testData = BufferUtils.getIncreasingByteArray(fileSize); + ByteArrayCacheManager manager = new ByteArrayCacheManager(); + sConf.set(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED, false); + //by default local cache fallback is not enabled, the read should fail for any error + LocalCacheFileInStream streamWithOutFallback = setupWithSingleFile(testData, manager); + try { + streamWithOutFallback.positionedRead(0, new byte[10], 100, 100); + fail("Expect position read fail here."); + } catch (ArrayIndexOutOfBoundsException e) { + //expected exception + } + sConf.set(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED, true); + LocalCacheFileInStream streamWithFallback = setupWithSingleFile(testData, manager); + Assert.assertEquals(100, streamWithFallback.positionedRead(0, new byte[10], 100, 100)); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_POSITION_READ_FALLBACK.getName()).getCount()); + } + + @Test + public void testPositionReadWithPerThreadContextSameThread() throws Exception + { + int fileSize = mPageSize * 5; + byte[] testData = BufferUtils.getIncreasingByteArray(fileSize); + ByteArrayCacheManager manager = new ByteArrayCacheManager(); + LocalCacheFileInStream stream = setupWithSingleFile(testData, manager); + + // read last page with cache enabled (cache miss) + CachePerThreadContext.get().setCacheEnabled(true); + stream.positionedRead(fileSize - mPageSize, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(0, + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + // read last page with cache enabled (cache hit) + stream.positionedRead(fileSize - mPageSize, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + + MetricsSystem.clearAllMetrics(); + // read first page with cache disabled (cache miss) + CachePerThreadContext.get().setCacheEnabled(false); + stream.positionedRead(0, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(0, + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + // read last page again with cache enabled (cache miss) + stream.positionedRead(0, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(2, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + } + private LocalCacheFileInStream setupWithSingleFile(byte[] data, CacheManager manager) throws Exception { Map files = new HashMap<>(); @@ -572,7 +676,8 @@ private LocalCacheFileInStream setupWithSingleFile(byte[] data, CacheManager man ByteArrayFileSystem fs = new ByteArrayFileSystem(files); return new LocalCacheFileInStream(fs.getStatus(testFilename), - (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf); + (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf, + Optional.empty()); } private Map setupWithMultipleFiles(Map files, @@ -587,7 +692,7 @@ private Map setupWithMultipleFiles(Map fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, - sConf)); + sConf, Optional.empty())); } catch (Exception e) { // skip } @@ -599,6 +704,7 @@ private URIStatus generateURIStatus(String path, long len) { FileInfo info = new FileInfo(); info.setFileId(path.hashCode()); info.setPath(path); + info.setUfsPath(path); info.setLength(len); return new URIStatus(info); } @@ -668,7 +774,7 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { if (!mPages.containsKey(pageId)) { return 0; @@ -678,6 +784,29 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return bytesToRead; } + @Override + public void commitFile(String fileId) { + throw new UnsupportedOperationException("commitFile method is unsupported. "); + } + + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + return bytesRead; + } + byte[] page = externalDataSupplier.get(); + if (page.length == 0) { + return 0; + } + buffer.writeBytes(page, pageOffset, bytesToRead); + put(pageId, page, cacheContext); + return bytesToRead; + } + @Override public boolean delete(PageId pageId) { return mPages.remove(pageId) != null; @@ -697,6 +826,49 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac public void close() throws Exception { // no-op } + + @Override + public void deleteFile(String fileId) { + // no-op + } + + @Override + public void deleteTempFile(String fileId) { + // no-op + } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + return Optional.empty(); + } + + class Usage implements CacheUsage { + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + + @Override + public long used() { + return mPages.values().stream().mapToInt(page -> page.length).sum(); + } + + @Override + public long available() { + return Integer.MAX_VALUE; + } + + @Override + public long capacity() { + return Integer.MAX_VALUE; + } + } } /** @@ -1036,7 +1208,7 @@ public TimedMockByteArrayCacheManager(StepTicker ticker) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { int read = super.get(pageId, pageOffset, bytesToRead, target, cacheContext); if (read > 0) { diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java index f564d4035ed2..0e3798f5d9cc 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java @@ -36,8 +36,13 @@ import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.NettyBufTargetBuffer; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.util.CommonUtils; import alluxio.util.WaitForOptions; import alluxio.util.io.BufferUtils; @@ -46,6 +51,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.DefaultFileRegion; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -669,6 +677,10 @@ public void asyncRestoreReadOnly() throws Exception { assertTrue(mCacheManager.delete(PAGE_ID2)); } + /** + * Invalid page file will be deleted and cache manager will start normally. + * @throws Exception + */ @Test public void syncRestoreUnknownFile() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_ASYNC_RESTORE_ENABLED, false); @@ -685,10 +697,14 @@ public void syncRestoreUnknownFile() throws Exception { mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); mCacheManager = LocalCacheManager.create(mCacheManagerOptions, mPageMetaStore); assertEquals(CacheManager.State.READ_WRITE, mCacheManager.state()); - assertEquals(0, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); - assertEquals(0, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); + assertEquals(PAGE1.length, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); + assertEquals(PAGE2.length, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); } + /** + * Invalid page file will be deleted and cache manager will start normally. + * @throws Exception + */ @Test public void asyncRestoreUnknownFile() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_ASYNC_RESTORE_ENABLED, true); @@ -703,8 +719,8 @@ public void asyncRestoreUnknownFile() throws Exception { FileUtils.createFile(Paths.get(rootDir, "invalidPageFile").toString()); mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); mCacheManager = createLocalCacheManager(mConf, mPageMetaStore); - assertEquals(0, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); - assertEquals(0, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); + assertEquals(PAGE1.length, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); + assertEquals(PAGE2.length, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); } @Test @@ -806,6 +822,29 @@ public void asyncRestoreWithMorePagesThanCapacity() throws Exception { assertEquals(0, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); } + @Test + public void ttlDeleteOldPagesWhenRestore() throws Exception { + mConf.set(PropertyKey.USER_CLIENT_CACHE_TTL_THRESHOLD_SECONDS, 5); + mConf.set(PropertyKey.USER_CLIENT_CACHE_TTL_ENABLED, true); + mCacheManagerOptions = CacheManagerOptions.create(mConf); + mCacheManager.close(); + PageStoreDir dir = PageStoreDir.createPageStoreDirs(mCacheManagerOptions) + .get(0); // previous page store has been closed + PageId pageUuid = new PageId(UUID.randomUUID().toString(), 0); + dir.getPageStore().put(PAGE_ID1, PAGE1); + dir.getPageStore().put(PAGE_ID2, PAGE2); + + dir = PageStoreDir.createPageStoreDirs(mCacheManagerOptions).get(0); + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + Thread.sleep(6000); + dir.getPageStore().put(pageUuid, + BufferUtils.getIncreasingByteArray(PAGE1.length + PAGE2.length + 1)); + mCacheManager = createLocalCacheManager(mConf, mPageMetaStore); + assertFalse(mCacheManager.hasPageUnsafe(PAGE_ID1)); + assertFalse(mCacheManager.hasPageUnsafe(PAGE_ID2)); + assertTrue(mCacheManager.hasPageUnsafe(pageUuid)); // we should have the new page + } + @Test public void asyncCache() throws Exception { // this must be smaller than the number of locks in the page store for the test to succeed @@ -965,6 +1004,58 @@ public void getTimeout() throws Exception { pageStore.setGetHanging(false); } + @Test + public void getFaultyReadWithNoExceptionManager() throws Exception { + PageStoreOptions pageStoreOptions = PageStoreOptions.create(mConf).get(0); + FaultyPageStore pageStore = new FaultyPageStore(); + PageStoreDir dir = + new LocalPageStoreDir(pageStoreOptions, pageStore, mEvictor); + + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + NoExceptionCacheManager cacheManager = + new NoExceptionCacheManager(createLocalCacheManager(mConf, mPageMetaStore)); + cacheManager.put(PAGE_ID1, PAGE1); + ByteArrayTargetBuffer targetBuffer = new ByteArrayTargetBuffer(mBuf, 0); + pageStore.setGetFaulty(true); + assertEquals(-1, cacheManager.get(PAGE_ID1, PAGE1.length, + targetBuffer, CacheContext.defaults())); + assertEquals(0, targetBuffer.offset()); + } + + @Test + public void getFaultyReadWithLocalCacheManager() throws Exception { + PageStoreOptions pageStoreOptions = PageStoreOptions.create(mConf).get(0); + FaultyPageStore pageStore = new FaultyPageStore(); + PageStoreDir dir = + new LocalPageStoreDir(pageStoreOptions, pageStore, mEvictor); + + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + LocalCacheManager cacheManager = createLocalCacheManager(mConf, mPageMetaStore); + cacheManager.put(PAGE_ID1, PAGE1); + ByteArrayTargetBuffer targetBuffer = new ByteArrayTargetBuffer(mBuf, 0); + pageStore.setGetFaulty(true); + assertEquals(-1, cacheManager.get(PAGE_ID1, PAGE1.length, + targetBuffer, CacheContext.defaults())); + assertEquals(0, targetBuffer.offset()); + } + + @Test + public void getCorruptedReadWithLocalCacheManager() throws Exception { + PageStoreOptions pageStoreOptions = PageStoreOptions.create(mConf).get(0); + FaultyPageStore pageStore = new FaultyPageStore(); + PageStoreDir dir = + new LocalPageStoreDir(pageStoreOptions, pageStore, mEvictor); + + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + LocalCacheManager cacheManager = createLocalCacheManager(mConf, mPageMetaStore); + cacheManager.put(PAGE_ID1, PAGE1); + ByteArrayTargetBuffer targetBuffer = new ByteArrayTargetBuffer(mBuf, 0); + pageStore.setGetCorrupted(true); + assertEquals(-1, cacheManager.get(PAGE_ID1, PAGE1.length, + targetBuffer, CacheContext.defaults())); + assertEquals(0, targetBuffer.offset()); + } + @Test public void deleteTimeout() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_TIMEOUT_DURATION, "2s"); @@ -1090,6 +1181,28 @@ public void listPageIds() throws Exception { 0).get(0)); } + @Test + public void getDataFileChannel() throws Exception { + mCacheManager = createLocalCacheManager(); + mCacheManager.put(PAGE_ID1, PAGE1); + CacheContext cacheContext = CacheContext.defaults(); + Optional dataFileChannel = mCacheManager.getDataFileChannel(PAGE_ID1, + 0, PAGE1.length, cacheContext); + assertNotNull(dataFileChannel); + assertEquals(dataFileChannel.isPresent(), true); + assertEquals(dataFileChannel.get().getNettyOutput() instanceof DefaultFileRegion, true); + DefaultFileRegion defaultFileRegion = + (DefaultFileRegion) dataFileChannel.get().getNettyOutput(); + ByteBuf buf = Unpooled.buffer(PAGE1.length); + NettyBufTargetBuffer targetBuffer = new NettyBufTargetBuffer(buf); + long bytesTransferred = defaultFileRegion.transferTo(targetBuffer.byteChannel(), 0); + assertEquals(bytesTransferred, PAGE1.length); + + byte[] bytes = new byte[PAGE1.length]; + buf.readBytes(bytes); + assertArrayEquals(PAGE1, bytes); + } + /** * A PageStore where put can throw IOException on put or delete. */ @@ -1100,6 +1213,23 @@ public FaultyPageStore() { private AtomicBoolean mPutFaulty = new AtomicBoolean(false); private AtomicBoolean mDeleteFaulty = new AtomicBoolean(false); + private AtomicBoolean mGetFaulty = new AtomicBoolean(false); + + private AtomicBoolean mGetCorrupted = new AtomicBoolean(false); + + @Override + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, + boolean isTemporary) throws IOException, PageNotFoundException { + if (mGetFaulty.get()) { + target.offset(target.offset() + 100); + throw new IOException("Page read fault"); + } + if (mGetCorrupted.get()) { + target.offset(target.offset() + 100); + throw new PageCorruptedException("page corrupted"); + } + return super.get(pageId, pageOffset, bytesToRead, target, isTemporary); + } @Override public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOException { @@ -1124,6 +1254,14 @@ void setPutFaulty(boolean faulty) { void setDeleteFaulty(boolean faulty) { mDeleteFaulty.set(faulty); } + + void setGetFaulty(boolean faulty) { + mGetFaulty.set(faulty); + } + + void setGetCorrupted(boolean faulty) { + mGetCorrupted.set(faulty); + } } /** diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java index 76cbae4ab79e..422517afad50 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java @@ -20,12 +20,12 @@ import static org.junit.Assert.fail; import alluxio.Constants; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; import alluxio.exception.PageNotFoundException; +import alluxio.file.ByteArrayTargetBuffer; import alluxio.util.io.BufferUtils; import org.junit.Before; diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java index 3c47646f64b9..3adaeda7f3ad 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java @@ -15,10 +15,13 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; +import alluxio.file.ByteArrayTargetBuffer; import org.junit.Before; import org.junit.Rule; @@ -170,12 +173,26 @@ public void cleanFileAndDirectory() throws Exception { assertFalse(Files.exists(p.getParent())); } + @Test + public void testCorruptedPages() throws Exception { + mOptions.setFileBuckets(1); + LocalPageStore pageStore = new LocalPageStore(mOptions); + byte[] buf = new byte[1000]; + PageId id = new PageId("1", 0); + pageStore.put(id, "corrupted".getBytes()); + assertThrows(PageCorruptedException.class, () -> { + //the bytes caller want to read is larger than the page file, mostly means the page corrupted + pageStore.get(id, 0, 100, new ByteArrayTargetBuffer(buf, 0)); + }); + } + private void helloWorldTest(PageStore store) throws Exception { String msg = "Hello, World!"; PageId id = new PageId("0", 0); store.put(id, msg.getBytes()); byte[] buf = new byte[1024]; - assertEquals(msg.getBytes().length, store.get(id, new ByteArrayTargetBuffer(buf, 0))); + assertEquals(msg.getBytes().length, store.get(id, 0, msg.length(), + new ByteArrayTargetBuffer(buf, 0))); assertArrayEquals(msg.getBytes(), Arrays.copyOfRange(buf, 0, msg.getBytes().length)); } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java index 44cae0a0234e..e20f11f50e5f 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java @@ -16,6 +16,7 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.file.ByteArrayTargetBuffer; import org.junit.Before; import org.junit.Test; @@ -42,7 +43,8 @@ private void helloWorldTest(PageStore store) throws Exception { PageId id = new PageId("0", 0); store.put(id, msg.getBytes()); byte[] buf = new byte[PAGE_SIZE]; - assertEquals(msg.getBytes().length, store.get(id, new ByteArrayTargetBuffer(buf, 0))); + assertEquals(msg.getBytes().length, + store.get(id, 0, msg.length(), new ByteArrayTargetBuffer(buf, 0))); assertArrayEquals(msg.getBytes(), Arrays.copyOfRange(buf, 0, msg.getBytes().length)); } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java index 3c6e768f5903..17e16c486c3b 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java @@ -11,11 +11,14 @@ package alluxio.client.file.cache.store; +import static alluxio.client.file.cache.CacheUsage.PartitionDescriptor.file; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import alluxio.ProjectConstants; import alluxio.client.file.cache.CacheManagerOptions; +import alluxio.client.file.cache.CacheUsage; +import alluxio.client.file.cache.CacheUsageView; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.conf.AlluxioConfiguration; @@ -34,17 +37,19 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.UUID; @RunWith(Parameterized.class) public class PageStoreDirTest { + public static final long CACHE_CAPACITY = 65536; + public static final long PAGE_SIZE = 1024; private final AlluxioConfiguration mConf = Configuration.global(); - @Parameterized.Parameters + @Parameterized.Parameters(name = "{index}-{0}") public static Collection data() { return Arrays.asList(new Object[][] { - {PageStoreType.ROCKS}, {PageStoreType.LOCAL}, {PageStoreType.MEM} }); @@ -65,8 +70,8 @@ public void before() throws Exception { CacheManagerOptions cacheManagerOptions = CacheManagerOptions.create(mConf); mOptions = cacheManagerOptions.getPageStoreOptions().get(0); mOptions.setStoreType(mPageStoreType); - mOptions.setPageSize(1024); - mOptions.setCacheSize(65536); + mOptions.setPageSize(PAGE_SIZE); + mOptions.setCacheSize(CACHE_CAPACITY); mOptions.setAlluxioVersion(ProjectConstants.VERSION); mOptions.setRootDir(Paths.get(mTemp.getRoot().getAbsolutePath())); @@ -120,4 +125,26 @@ public void getPagesUUID() throws Exception { assertEquals(pages, restored); } } + + @Test + public void cacheUsage() throws Exception { + int len = 32; + int count = 16; + byte[] data = BufferUtils.getIncreasingByteArray(len); + for (int i = 0; i < count; i++) { + PageId id = new PageId("0", i); + mPageStoreDir.getPageStore().put(id, data); + mPageStoreDir.putPage(new PageInfo(id, data.length, mPageStoreDir)); + } + Optional usage = mPageStoreDir.getUsage(); + assertEquals(Optional.of(mPageStoreDir.getCapacityBytes()), + usage.map(CacheUsageView::capacity)); + assertEquals(Optional.of((long) len * count), usage.map(CacheUsageView::used)); + assertEquals(Optional.of(mPageStoreDir.getCapacityBytes() - (long) len * count), + usage.map(CacheUsageView::available)); + // cache dir currently does not support get file level usage stat + Optional fileUsage = mPageStoreDir.getUsage() + .flatMap(usage1 -> usage1.partitionedBy(file("0"))); + assertEquals(Optional.empty(), fileUsage); + } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java index e4b5cc115159..3c382d5b4f22 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java @@ -19,7 +19,9 @@ import alluxio.ProjectConstants; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; +import alluxio.file.ByteArrayTargetBuffer; import alluxio.util.io.BufferUtils; import org.junit.After; @@ -40,7 +42,6 @@ public class PageStoreTest { @Parameterized.Parameters public static Collection data() { return Arrays.asList(new Object[][] { - {PageStoreType.ROCKS}, {PageStoreType.LOCAL}, {PageStoreType.MEM} }); @@ -79,7 +80,8 @@ public void helloWorldTest() throws Exception { PageId id = new PageId("0", 0); mPageStore.put(id, msgBytes); byte[] buf = new byte[1024]; - assertEquals(msgBytes.length, mPageStore.get(id, new ByteArrayTargetBuffer(buf, 0))); + assertEquals(msgBytes.length, + mPageStore.get(id, 0, msgBytes.length, new ByteArrayTargetBuffer(buf, 0))); assertArrayEquals(msgBytes, Arrays.copyOfRange(buf, 0, msgBytes.length)); mPageStore.delete(id); try { @@ -97,7 +99,8 @@ public void getOffset() throws Exception { mPageStore.put(id, BufferUtils.getIncreasingByteArray(len)); byte[] buf = new byte[len]; for (int offset = 1; offset < len; offset++) { - int bytesRead = mPageStore.get(id, offset, len, new ByteArrayTargetBuffer(buf, 0), false); + int bytesRead = mPageStore.get(id, offset, len - offset, + new ByteArrayTargetBuffer(buf, 0), false); assertEquals(len - offset, bytesRead); assertArrayEquals(BufferUtils.getIncreasingByteArray(offset, len - offset), Arrays.copyOfRange(buf, 0, bytesRead)); @@ -111,7 +114,7 @@ public void getOffsetOverflow() throws Exception { PageId id = new PageId("0", 0); mPageStore.put(id, BufferUtils.getIncreasingByteArray(len)); byte[] buf = new byte[1024]; - assertThrows(IllegalArgumentException.class, () -> + assertThrows(PageCorruptedException.class, () -> mPageStore.get(id, offset, len, new ByteArrayTargetBuffer(buf, 0))); } diff --git a/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java b/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java index 5b0c74fd6fe7..372bc3604033 100644 --- a/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java +++ b/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java @@ -16,12 +16,14 @@ import alluxio.AlluxioURI; import alluxio.client.file.CacheContext; +import alluxio.client.file.FileInStream; import alluxio.client.file.URIStatus; import alluxio.client.file.cache.CacheManager; import alluxio.client.file.cache.LocalCacheFileInStream; import alluxio.client.file.cache.filter.CacheFilter; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.PropertyKey; +import alluxio.exception.AlluxioException; import alluxio.metrics.MetricsConfig; import alluxio.metrics.MetricsSystem; import alluxio.wire.FileInfo; @@ -41,6 +43,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; +import java.util.Optional; import java.util.Properties; /** @@ -93,8 +96,8 @@ public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuratio } MetricsSystem.startSinksFromConfig(new MetricsConfig(metricsProperties)); mCacheManager = CacheManager.Factory.get(mAlluxioConf); - LocalCacheFileInStream.registerMetrics(); mCacheFilter = CacheFilter.create(mAlluxioConf); + LocalCacheFileInStream.registerMetrics(); } @Override @@ -146,19 +149,45 @@ public FSDataInputStream open(Path path, int bufferSize) throws IOException { } /** - * Attempts to open the specified file for reading. + * A wrapper method to default not enforce an open call. * * @param status the status of the file to open * @param bufferSize stream buffer size in bytes, currently unused * @return an {@link FSDataInputStream} at the indicated path of a file */ public FSDataInputStream open(URIStatus status, int bufferSize) throws IOException { + return open(status, bufferSize, false); + } + + /** + * Attempts to open the specified file for reading. + * + * @param status the status of the file to open + * @param bufferSize stream buffer size in bytes, currently unused + * @param enforceOpen flag to enforce calling open to external storage + * @return an {@link FSDataInputStream} at the indicated path of a file + */ + public FSDataInputStream open(URIStatus status, int bufferSize, boolean enforceOpen) + throws IOException { if (mCacheManager == null || !mCacheFilter.needsCache(status)) { return mExternalFileSystem.open(HadoopUtils.toPath(new AlluxioURI(status.getPath())), bufferSize); } + Optional externalFileInStream; + if (enforceOpen) { + try { + // making the open call right now, instead of later when called back + externalFileInStream = Optional.of(mAlluxioFileOpener.open(status)); + } catch (AlluxioException e) { + throw new IOException(e); + } + } else { + externalFileInStream = Optional.empty(); + } + return new FSDataInputStream(new HdfsFileInputStream( - new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf), + new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf, + externalFileInStream), statistics)); } diff --git a/core/common/pom.xml b/core/common/pom.xml index 7834960d54d4..c4082e137d33 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -95,10 +95,26 @@ io.grpc grpc-stub + + io.prometheus + prometheus-metrics-core + + + io.prometheus + prometheus-metrics-exporter-servlet-jakarta + + + io.prometheus + prometheus-metrics-instrumentation-jvm + io.swagger swagger-annotations + + jakarta.servlet + jakarta.servlet-api + org.apache.commons commons-lang3 @@ -119,6 +135,10 @@ org.apache.zookeeper zookeeper + + org.eclipse.jetty + jetty-servlet + org.reflections reflections diff --git a/core/common/src/main/java/alluxio/conf/PropertyKey.java b/core/common/src/main/java/alluxio/conf/PropertyKey.java index dbf013f7b959..ca03396c0300 100755 --- a/core/common/src/main/java/alluxio/conf/PropertyKey.java +++ b/core/common/src/main/java/alluxio/conf/PropertyKey.java @@ -6042,6 +6042,13 @@ public String toString() { .setConsistencyCheckLevel(ConsistencyCheckLevel.WARN) .setScope(Scope.CLIENT) .build(); + public static final PropertyKey USER_CLIENT_CACHE_FALLBACK_ENABLED = + booleanBuilder(Name.USER_CLIENT_CACHE_FALLBACK_ENABLED) + .setDefaultValue(false) + .setDescription("If this is enabled, local cache will fallback to external storage.") + .setConsistencyCheckLevel(ConsistencyCheckLevel.WARN) + .setScope(Scope.CLIENT) + .build(); public static final PropertyKey USER_CLIENT_CACHE_FILTER_CLASS = classBuilder(Name.USER_CLIENT_CACHE_FILTER_CLASS) .setDefaultValue("alluxio.client.file.cache.filter.DefaultCacheFilter") @@ -8889,6 +8896,8 @@ public static final class Name { "alluxio.user.client.cache.async.write.threads"; public static final String USER_CLIENT_CACHE_ENABLED = "alluxio.user.client.cache.enabled"; + public static final String USER_CLIENT_CACHE_FALLBACK_ENABLED = + "alluxio.user.client.cache.fallback.enabled"; public static final String USER_CLIENT_CACHE_FILTER_CLASS = "alluxio.user.client.cache.filter.class"; public static final String USER_CLIENT_CACHE_FILTER_CONFIG_FILE = diff --git a/core/common/src/main/java/alluxio/exception/PageCorruptedException.java b/core/common/src/main/java/alluxio/exception/PageCorruptedException.java new file mode 100644 index 000000000000..45f4be35cac9 --- /dev/null +++ b/core/common/src/main/java/alluxio/exception/PageCorruptedException.java @@ -0,0 +1,35 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.exception; + +/** + * An exception that should be thrown when the data of a page has been corrupted in store. + */ +public class PageCorruptedException extends RuntimeException { + + /** + * Construct PageCorruptedException with the specified message. + * @param message + */ + public PageCorruptedException(String message) { + super(message); + } + + /** + * Construct PageCorruptedException with the specified message and cause. + * @param message + * @param cause + */ + public PageCorruptedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java b/core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java new file mode 100644 index 000000000000..4dd4d5360e62 --- /dev/null +++ b/core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java @@ -0,0 +1,105 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import alluxio.annotation.SuppressFBWarnings; +import alluxio.util.io.ChannelAdapters; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +/** + * Target buffer backed by bytes array for zero-copy read from page store. + */ +@SuppressFBWarnings( + value = "EI_EXPOSE_REP2", + justification = "The target byte array is exposed as we expect.") +public class ByteArrayTargetBuffer implements ReadTargetBuffer { + private final byte[] mTarget; + private int mOffset; + + /** + * Constructor. + * @param target + * @param offset + */ + public ByteArrayTargetBuffer(byte[] target, int offset) { + mTarget = target; + mOffset = offset; + } + + @Override + public byte[] byteArray() { + return mTarget; + } + + @Override + public ByteBuffer byteBuffer() { + return ByteBuffer.wrap(mTarget); + } + + @Override + public int offset() { + return mOffset; + } + + @Override + public void offset(int newOffset) { + mOffset = newOffset; + } + + @Override + public long remaining() { + return mTarget.length - mOffset; + } + + @Override + public void writeBytes(byte[] srcArray, int srcOffset, int length) { + System.arraycopy(srcArray, srcOffset, mTarget, mOffset, length); + mOffset += length; + } + + @Override + public void writeBytes(ByteBuf buf) { + int bytesToRead = Math.min(buf.readableBytes(), mTarget.length - mOffset); + buf.readBytes(mTarget, mOffset, bytesToRead); + mOffset += bytesToRead; + } + + @Override + public int readFromFile(RandomAccessFile file, int length) throws IOException { + int bytesRead = file.read(mTarget, mOffset, length); + if (bytesRead != -1) { + mOffset += bytesRead; + } + return bytesRead; + } + + @Override + public int readFromInputStream(InputStream is, int length) throws IOException { + int bytesRead = is.read(mTarget, mOffset, length); + if (bytesRead != -1) { + mOffset += bytesRead; + } + return bytesRead; + } + + @Override + public WritableByteChannel byteChannel() { + return ChannelAdapters.intoByteArray(mTarget, mOffset, mTarget.length - mOffset); + } +} diff --git a/core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java b/core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java new file mode 100644 index 000000000000..20734be67cdf --- /dev/null +++ b/core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java @@ -0,0 +1,114 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import alluxio.util.io.ChannelAdapters; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Target buffer backed by nio ByteBuffer for zero-copy read from page store. + */ +public class ByteBufferTargetBuffer implements ReadTargetBuffer { + private final ByteBuffer mTarget; + + /** + * Constructor. + * @param target + */ + public ByteBufferTargetBuffer(ByteBuffer target) { + mTarget = target; + } + + @Override + public byte[] byteArray() { + if (mTarget.hasArray()) { + return mTarget.array(); + } + throw new UnsupportedOperationException("ByteBuffer is not backed by an array"); + } + + @Override + public ByteBuffer byteBuffer() { + return mTarget; + } + + @Override + public int offset() { + return mTarget.position(); + } + + @Override + public void offset(int newOffset) { + mTarget.position(newOffset); + } + + @Override + public WritableByteChannel byteChannel() { + return ChannelAdapters.intoByteBuffer(mTarget); + } + + @Override + public long remaining() { + return mTarget.remaining(); + } + + @Override + public void writeBytes(byte[] srcArray, int srcOffset, int length) { + mTarget.put(srcArray, srcOffset, length); + } + + @Override + public void writeBytes(ByteBuf buf) { + if (mTarget.remaining() <= buf.readableBytes()) { + buf.readBytes(mTarget); + return; + } + int oldLimit = mTarget.limit(); + mTarget.limit(mTarget.position() + buf.readableBytes()); + buf.readBytes(mTarget); + mTarget.limit(oldLimit); + } + + @Override + public int readFromFile(RandomAccessFile file, int length) throws IOException { + int bytesToRead = Math.min(length, mTarget.remaining()); + ByteBuffer slice = mTarget.slice(); + slice.limit(bytesToRead); + int bytesRead = file.getChannel().read(slice); + if (bytesRead > 0) { + mTarget.position(mTarget.position() + bytesRead); + } + return bytesRead; + } + + @Override + public int readFromInputStream(InputStream is, int length) throws IOException { + int bytesToRead = Math.min(length, mTarget.remaining()); + ReadableByteChannel source = Channels.newChannel(is); + ByteBuffer slice = mTarget.slice(); + slice.limit(bytesToRead); + int bytesRead = source.read(slice); + if (bytesRead > 0) { + mTarget.position(mTarget.position() + bytesRead); + } + return bytesRead; + } +} diff --git a/core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java b/core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java new file mode 100644 index 000000000000..0dc2b5f219d6 --- /dev/null +++ b/core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java @@ -0,0 +1,120 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Netty Buf backed target buffer for zero-copy read from page store. + */ +public class NettyBufTargetBuffer implements ReadTargetBuffer { + private final ByteBuf mTarget; + + /** + * @param target target buffer + */ + public NettyBufTargetBuffer(ByteBuf target) { + mTarget = target; + } + + @Override + public byte[] byteArray() { + return mTarget.array(); + } + + @Override + public ByteBuffer byteBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public int offset() { + return mTarget.writerIndex(); + } + + @Override + public void offset(int newOffset) { + mTarget.writerIndex(newOffset); + } + + @Override + public WritableByteChannel byteChannel() { + return new WritableByteChannel() { + @Override + public int write(ByteBuffer src) throws IOException { + int readableBytes = src.remaining(); + mTarget.writeBytes(src); + return readableBytes - src.remaining(); + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() throws IOException { + } + }; + } + + @Override + public long remaining() { + return mTarget.writableBytes(); + } + + @Override + public void writeBytes(byte[] srcArray, int srcOffset, int length) { + mTarget.writeBytes(srcArray, srcOffset, length); + } + + @Override + public void writeBytes(ByteBuf buf) { + mTarget.writeBytes(buf); + } + + @Override + public int readFromFile(RandomAccessFile file, int length) throws IOException { + try (FileChannel channel = file.getChannel()) { + return mTarget.writeBytes(channel, length); + } + } + + @Override + public int readFromInputStream(InputStream is, int length) throws IOException { + int bytesToRead = Math.min(length, mTarget.writableBytes()); + ReadableByteChannel source = Channels.newChannel(is); + ByteBuffer slice = mTarget.nioBuffer(mTarget.writerIndex(), bytesToRead); + int bytesRead = source.read(slice); + if (bytesRead > 0) { + mTarget.writerIndex(mTarget.writerIndex() + bytesRead); + } + return bytesRead; + } + + /** + * Get the internal Bytebuf object. + * @return the internal Bytebuf object + */ + public ByteBuf getTargetBuffer() { + return mTarget; + } +} diff --git a/core/common/src/main/java/alluxio/file/ReadTargetBuffer.java b/core/common/src/main/java/alluxio/file/ReadTargetBuffer.java new file mode 100644 index 000000000000..97ec000baaa8 --- /dev/null +++ b/core/common/src/main/java/alluxio/file/ReadTargetBuffer.java @@ -0,0 +1,86 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +/** + * TargetBuffer for zero-copy read from page store. + */ +public interface ReadTargetBuffer { + + /** + * @return the byte array + */ + byte[] byteArray(); + + /** + * + * @return the byte buffer + */ + ByteBuffer byteBuffer(); + + /** + * @return offset in the buffer + */ + int offset(); + + /** + * Sets the new offset in the buffer. + * + * @param newOffset the new offset + */ + void offset(int newOffset); + + /** + * @return the writable channel + */ + WritableByteChannel byteChannel(); + + /** + * @return the remaining for this buffer + */ + long remaining(); + + /** + * @param srcArray + * @param srcOffset + * @param length + */ + void writeBytes(byte[] srcArray, int srcOffset, int length); + + /** + * @param buf + */ + void writeBytes(ByteBuf buf); + + /** + * @param file + * @param length + * @return bytes read from the file + */ + int readFromFile(RandomAccessFile file, int length) throws IOException; + + /** + * @param is + * @param length + * @return bytes read from input stream + * @throws IOException + */ + int readFromInputStream(InputStream is, int length) throws IOException; +} diff --git a/core/common/src/main/java/alluxio/metrics/MetricKey.java b/core/common/src/main/java/alluxio/metrics/MetricKey.java index 0f9100f9d2d1..5dc399797b80 100644 --- a/core/common/src/main/java/alluxio/metrics/MetricKey.java +++ b/core/common/src/main/java/alluxio/metrics/MetricKey.java @@ -2398,6 +2398,12 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.METER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_EXTERNAL_REQUESTS = + new Builder("Client.CacheExternalRequests") + .setDescription("Total number of requests to read from external storage.") + .setMetricType(MetricType.COUNTER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS = new Builder("Client.CachePageReadCacheTimeNanos") .setDescription("Time in nanoseconds taken to read a page from the client cache " @@ -2412,12 +2418,24 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.METER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_POSITION_READ_FALLBACK = + new Builder("Client.CacheBytesPositionReadFallback") + .setDescription("Total number of position read fallback to external storage.") + .setMetricType(MetricType.COUNTER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_BYTES_DISCARDED = new Builder("Client.CacheBytesDiscarded") .setDescription("Total number of bytes discarded when restoring the page store.") .setMetricType(MetricType.METER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_PAGES_INVALIDATED = + new Builder("Client.CachePagesInvalidated") + .setDescription("Total number of pages invalidated by TTL rules") + .setMetricType(MetricType.METER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_BYTES_EVICTED = new Builder("Client.CacheBytesEvicted") .setDescription("Total number of bytes evicted from the client cache.") @@ -2430,6 +2448,12 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.COUNTER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_PAGES_AGES = + new Builder("Client.CachePagesAges") + .setDescription("The ages of pages in the client cache.") + .setMetricType(MetricType.HISTOGRAM) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_PAGES_DISCARDED = new Builder("Client.CachePagesDiscarded") .setDescription("Total number of pages discarded when restoring the page store.") @@ -2454,6 +2478,12 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.GAUGE) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_HIT_REQUESTS = + new Builder("Client.CacheHitRequests") + .setDescription("Total number of requests of hitting the cache.") + .setMetricType(MetricType.COUNTER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_SPACE_AVAILABLE = new Builder("Client.CacheSpaceAvailable") .setDescription("Amount of bytes available in the client cache.") diff --git a/core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java b/core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java new file mode 100644 index 000000000000..12d8996e7370 --- /dev/null +++ b/core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java @@ -0,0 +1,311 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.metrics; + +import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; +import alluxio.util.CommonUtils; +import alluxio.util.FormatUtils; + +import io.prometheus.metrics.config.PrometheusProperties; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.core.metrics.GaugeWithCallback; +import io.prometheus.metrics.core.metrics.Histogram; +import io.prometheus.metrics.exporter.common.PrometheusHttpRequest; +import io.prometheus.metrics.exporter.common.PrometheusHttpResponse; +import io.prometheus.metrics.exporter.common.PrometheusScrapeHandler; +import io.prometheus.metrics.exporter.servlet.jakarta.HttpExchangeAdapter; +import io.prometheus.metrics.instrumentation.jvm.JvmMetrics; +import io.prometheus.metrics.model.registry.PrometheusRegistry; +import io.prometheus.metrics.model.snapshots.Unit; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Enumeration; +import java.util.List; +import java.util.function.DoubleSupplier; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * A metrics system for all processes to define and collect all the metrics, + * and expose all the metrics to the web server. + */ +public final class MultiDimensionalMetricsSystem { + public static final Histogram DATA_ACCESS = Histogram.builder() + .name("alluxio_data_access") + .help("aggregated throughput of all the data access") + .unit(Unit.BYTES) + .labelNames("method") + .build(); + + public static final Counter META_OPERATION = Counter.builder() + .name("alluxio_meta_operation") + .help("counter of rpc calls of the meta operations") + .labelNames("op") + .build(); + + public static final Counter UFS_DATA_ACCESS = Counter.builder() + .name("alluxio_ufs_data_access") + .help("amount of the ufs access") + .unit(Unit.BYTES) + .labelNames("method") + .build(); + + public static final Counter CACHED_DATA_READ = Counter.builder() + .name("alluxio_cached_data_read") + .help("amount of the read cached data") + .unit(Unit.BYTES) + .build(); + + public static final Counter EXTERNAL_DATA_READ = Counter.builder() + .name("alluxio_external_data_read") + .help("amount of the read data when cache missed on client") + .unit(Unit.BYTES) + .build(); + + public static final Counter CACHED_EVICTED_DATA = Counter.builder() + .name("alluxio_cached_evicted_data") + .help("amount of the evicted data") + .unit(Unit.BYTES) + .build(); + + public static final DoubleSupplier NULL_SUPPLIER = () -> 0; + private static DoubleSupplier sCacheStorageSupplier = NULL_SUPPLIER; + private static DoubleSupplier sCacheManagerLoadingPagesSupplier = NULL_SUPPLIER; + + public static final GaugeWithCallback CACHED_STORAGE = GaugeWithCallback.builder() + .name("alluxio_cached_storage") + .help("amount of the cached data") + .unit(Unit.BYTES) + .callback(callback -> callback.call(sCacheStorageSupplier.getAsDouble())) + .build(); + + public static final GaugeWithCallback CACHED_CAPACITY = GaugeWithCallback.builder() + .name("alluxio_cached_capacity") + .help("configured maximum cache storage") + .unit(Unit.BYTES) + .callback(callback -> { + List sizes = Configuration.global().getList(PropertyKey.WORKER_PAGE_STORE_SIZES); + long sum = sizes.stream().map(FormatUtils::parseSpaceSize).reduce(0L, Long::sum); + callback.call(sum); + }) + .build(); + public static final GaugeWithCallback CACHE_MANAGER_LOADING_PAGES = GaugeWithCallback.builder() + .name("alluxio_cached_manager_loading_pages") + .help("number of pages being loaded from UFS into the cache manager") + .unit(Unit.BYTES) + .callback(callback -> callback.call(sCacheManagerLoadingPagesSupplier.getAsDouble())) + .build(); + + // Distributed load related + public static final Counter DISTRIBUTED_LOAD_JOB_FAILURE = Counter.builder() + .name("alluxio_distributed_load_job_failure") + .help("counter of the distributed load failure received on master") + .labelNames("reason", "final_attempt", "worker") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_SCANNED = Counter.builder() + .name("alluxio_distributed_load_job_scanned") + .help("counter of the inodes scanned in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_PROCESSED = Counter.builder() + .name("alluxio_distributed_load_job_processed") + .help("counter of the non empty file copies loaded in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_SKIPPED = Counter.builder() + .name("alluxio_distributed_load_job_skipped") + .help("counter of the inodes skipped in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_LOADED_BYTES = Counter.builder() + .name("alluxio_distributed_load_job_loaded_bytes") + .help("counter of the bytes loaded in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_DISPATCHED_SIZE = Counter.builder() + .name("alluxio_distributed_load_job_dispatched_size") + .help("distributed_load_job_batch_size") + .labelNames("worker") + .register(); + public static final Counter DISTRIBUTED_LOAD_WORKER_SUBTASKS_TOTAL = Counter.builder() + .name("alluxio_distributed_worker_subtasks_total") + .help("counter of the total subtasks a worker processes") + .register(); + public static final Counter DISTRIBUTED_LOAD_WORKER_BYTES_LOADED_TOTAL = Counter.builder() + .name("alluxio_distributed_worker_bytes_loaded_total") + .unit(Unit.BYTES) + .help("counter of the total bytes loaded in distributed load of a worker") + .register(); + public static final Counter DISTRIBUTED_LOAD_WORKER_ERRORS_TOTAL = Counter.builder() + .name("alluxio_distributed_worker_errors_total") + .help("counter of the total errors in distributed load of a worker") + .register(); + + // Client fallback related + public static final Counter CLIENT_RPC_RETRY_ON_DIFFERENT_WORKERS = Counter.builder() + .name("client_rpc_retry_on_different_workers") + .help("counter of client retry on different workers if multi replica is enabled") + .labelNames("op", "retry_count") + .build(); + + /** + * Initialize all the metrics. + */ + public static void initMetrics() { + JvmMetrics.builder().register(); + if (CommonUtils.PROCESS_TYPE.get() != CommonUtils.ProcessType.WORKER + && CommonUtils.PROCESS_TYPE.get() != CommonUtils.ProcessType.CLIENT) { + return; + } + if (CommonUtils.PROCESS_TYPE.get() == CommonUtils.ProcessType.CLIENT) { + PrometheusRegistry.defaultRegistry.register(EXTERNAL_DATA_READ); + } + PrometheusRegistry.defaultRegistry.register(DATA_ACCESS); + PrometheusRegistry.defaultRegistry.register(UFS_DATA_ACCESS); + PrometheusRegistry.defaultRegistry.register(META_OPERATION); + PrometheusRegistry.defaultRegistry.register(CACHED_DATA_READ); + PrometheusRegistry.defaultRegistry.register(CACHED_EVICTED_DATA); + PrometheusRegistry.defaultRegistry.register(CACHED_STORAGE); + PrometheusRegistry.defaultRegistry.register(CACHED_CAPACITY); + PrometheusRegistry.defaultRegistry.register(CACHE_MANAGER_LOADING_PAGES); + } + + /** + * Set the supplier for CACHE_STORAGE metrics. + * + * @param supplier supplier for cache storage + */ + public static void setCacheStorageSupplier(DoubleSupplier supplier) { + sCacheStorageSupplier = supplier; + } + + /** + * @param supplier the supplier for CACHE_MANAGER_LOADING_PAGES + */ + public static void setCacheManagerLoadingPagesSupplier( + DoubleSupplier supplier) { + MultiDimensionalMetricsSystem.sCacheManagerLoadingPagesSupplier = supplier; + } + + /** + * A servlet that exposes metrics data in prometheus format by HTTP. + */ + public static class WebHandler extends HttpServlet { + private static final long serialVersionUID = 1; + private static final String SERVLET_PATH = "/metrics"; + + private final PrometheusScrapeHandler mHandler = new PrometheusScrapeHandler( + PrometheusProperties.get(), PrometheusRegistry.defaultRegistry); + + /** + * Returns the servlet handler for Prometheus exposure. + * + * @return servlet handler + */ + public static ServletContextHandler getHandler() { + ServletContextHandler contextHandler = new ServletContextHandler(); + contextHandler.setContextPath(SERVLET_PATH); + contextHandler.addServlet(new ServletHolder(new WebHandler()), "/"); + return contextHandler; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + mHandler.handleRequest(new ServletAdapter(req, resp)); + } + } + + /** + * An adapter to make the javax.servlet and jakarta.servlet compatible. + */ + private static class ServletAdapter extends HttpExchangeAdapter { + private final Request mRequest; + private final Response mResponse; + + public ServletAdapter(HttpServletRequest request, HttpServletResponse response) { + super(null, null); + mRequest = new Request(request); + mResponse = new Response(response); + } + + @Override + public PrometheusHttpRequest getRequest() { + return mRequest; + } + + @Override + public PrometheusHttpResponse getResponse() { + return mResponse; + } + + @Override + public void handleException(IOException e) throws IOException { + throw e; + } + + @Override + public void handleException(RuntimeException e) { + throw e; + } + + @Override + public void close() {} + + private static class Request implements PrometheusHttpRequest { + private final HttpServletRequest mRequest; + + public Request(HttpServletRequest request) { + mRequest = request; + } + + @Override + public String getQueryString() { + return mRequest.getQueryString(); + } + + @Override + public Enumeration getHeaders(String name) { + return mRequest.getHeaders(name); + } + + @Override + public String getMethod() { + return mRequest.getMethod(); + } + } + + private static class Response implements PrometheusHttpResponse { + private final HttpServletResponse mResponse; + + public Response(HttpServletResponse response) { + mResponse = response; + } + + @Override + public void setHeader(String name, String value) { + mResponse.setHeader(name, value); + } + + @Override + public OutputStream sendHeadersAndGetBody( + int statusCode, int contentLength) throws IOException { + if (mResponse.getHeader("Content-Length") == null && contentLength > 0) { + mResponse.setContentLength(contentLength); + } + + mResponse.setStatus(statusCode); + return mResponse.getOutputStream(); + } + } + } +} diff --git a/core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java b/core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java new file mode 100644 index 000000000000..5a0cab898826 --- /dev/null +++ b/core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java @@ -0,0 +1,86 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.network.protocol.databuffer; + +import com.google.common.base.Preconditions; +import io.netty.channel.DefaultFileRegion; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +/** + * A DataBuffer with the underlying data being a {@link FileChannel}. + */ +public final class DataFileChannel implements DataBuffer { + private final File mFile; + private final long mOffset; + private final long mLength; + + /** + * + * @param file The file + * @param offset The offset into the FileChannel + * @param length The length of the data to read + */ + public DataFileChannel(File file, long offset, long length) { + mFile = Preconditions.checkNotNull(file, "file"); + mOffset = offset; + mLength = length; + } + + @Override + public Object getNettyOutput() { + return new DefaultFileRegion(mFile, mOffset, mLength); + } + + @Override + public long getLength() { + return mLength; + } + + @Override + public ByteBuffer getReadOnlyByteBuffer() { + throw new UnsupportedOperationException( + "DataFileChannel#getReadOnlyByteBuffer is not implemented."); + } + + @Override + public void readBytes(byte[] dst, int dstIndex, int length) { + throw new UnsupportedOperationException("DataFileChannel#readBytes is not implemented."); + } + + @Override + public void readBytes(OutputStream outputStream, int length) throws IOException { + throw new UnsupportedOperationException("DataFileChannel#readBytes is not implemented."); + } + + @Override + public void readBytes(ByteBuffer outputBuf) { + throw new UnsupportedOperationException("DataFileChannel#readBytes is not implemented."); + } + + @Override + public int readableBytes() { + int lengthInt = (int) mLength; + Preconditions.checkArgument(mLength == (long) lengthInt, + "size of file %s is %s, cannot be cast to int", mFile, mLength); + return lengthInt; + } + + @Override + public void release() { + // Nothing we need to release explicitly, let GC take care of all objects. + } +} diff --git a/core/common/src/main/java/alluxio/util/io/ChannelAdapters.java b/core/common/src/main/java/alluxio/util/io/ChannelAdapters.java new file mode 100644 index 000000000000..d28a51fd64eb --- /dev/null +++ b/core/common/src/main/java/alluxio/util/io/ChannelAdapters.java @@ -0,0 +1,141 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.util.io; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.WritableByteChannel; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Channel adapters. + *
    + *

    Thread Safety

    + * The channel implementations in this class are thread safe, as per the general requirement + * of the {@link java.nio.channels.Channel} interface. However, in case a channel is based on + * an underlying buffer, cautions must be taken to preserve thread safety: + *
      + *
    • Two channels should not share the same underlying buffer and be operated on + * concurrently.
      • + *
    • A buffer should not be manipulated directly while there is an outstanding channel that + * uses it as the underlying buffer.
      • + *
    + */ +public class ChannelAdapters { + private ChannelAdapters() { } // utility class + + /** + * Adapts a ByteBuffer to be the destination of a {@link WritableByteChannel}. + *
    + * The returned channel, when its {@link WritableByteChannel#write(ByteBuffer)} method is called, + * will copy {@code p} bytes from the source buffer to the output buffer, + * where {@code p = min(src.remaining(), output.remaining())}. After the write is successful, + * the positions of both the source and output buffer will have increased by {@code p}. + * If the output buffer has no space remaining, while the source + * buffer has available bytes, {@link ClosedChannelException} is thrown. + * + * @param output the output byte buffer + * @return channel + */ + public static WritableByteChannel intoByteBuffer(ByteBuffer output) { + return new ByteBufferWritableChannel(output); + } + + /** + * Adapts a byte array to be the destination of a {@link WritableByteChannel}. + * Bytes will be written from the beginning of the array and to the end. + * + * @param output output buffer + * @return channel + */ + public static WritableByteChannel intoByteArray(byte[] output) { + return intoByteBuffer(ByteBuffer.wrap(output)); + } + + /** + * Adapts a byte array to be the destination of a {@link WritableByteChannel}. + * Bytes will be written from index {@code offset} to {@code offset + length}. + * + * @param output output buffer + * @param offset offset within the buffer + * @param length length of the writable region within the buffer + * @return channel + */ + public static WritableByteChannel intoByteArray(byte[] output, int offset, int length) { + return intoByteBuffer(ByteBuffer.wrap(output, offset, length)); + } + + /** + * todo(bowen): implement. + * @param output output + * @return channel + */ + public static WritableByteChannel intoByteBuf(ByteBuf output) { + throw new UnsupportedOperationException("todo"); + } + + @ThreadSafe + static class ByteBufferWritableChannel implements WritableByteChannel { + @GuardedBy("this") + private final ByteBuffer mOutput; + private boolean mClosed = false; + + ByteBufferWritableChannel(ByteBuffer byteBuffer) { + mOutput = byteBuffer; + } + + /** + * Copies bytes from the source buffer into the channel. + * + * @param src the buffer from which bytes are to be retrieved + * @return number of bytes written, 0 when the source buffer has no bytes remaining + * @throws ClosedChannelException if the channel has been explicitly closed, + * or when the output buffer does not have remaining space, + * but the source buffer has remaining readable bytes + */ + @Override + public int write(ByteBuffer src) throws IOException { + if (mClosed) { + throw new ClosedChannelException(); + } + synchronized (this) { + int srcRemaining = src.remaining(); + int outputRemaining = mOutput.remaining(); + if (outputRemaining == 0 && srcRemaining > 0) { + throw new ClosedChannelException(); + } + int bytesToCopy = Math.min(srcRemaining, outputRemaining); + ByteBuffer slice = src.slice(); + slice.limit(bytesToCopy); + mOutput.put(slice); + src.position(src.position() + bytesToCopy); + return bytesToCopy; + } + } + + @Override + public boolean isOpen() { + return !mClosed; + } + + @Override + public void close() throws IOException { + if (!mClosed) { + mClosed = true; + } + } + } +} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java index 1564190d76b3..f1e9d1908744 100644 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java +++ b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java @@ -11,6 +11,7 @@ package alluxio.worker.page; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.DefaultPageMetaStore; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; @@ -21,6 +22,7 @@ import alluxio.client.quota.CacheScope; import alluxio.collections.IndexDefinition; import alluxio.collections.IndexedSet; +import alluxio.exception.FileDoesNotExistException; import alluxio.exception.PageNotFoundException; import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; import alluxio.worker.block.BlockStoreEventListener; @@ -217,21 +219,6 @@ public void addPage(PageId pageId, PageInfo pageInfo) { mDelegate.addPage(pageId, pageInfo); } - /** - * Gets the block meta for a page of the block. - * @param pageId the page ID - * @return block meta - * @throws BlockDoesNotExistRuntimeException when the block is not being stored in the store - */ - private PagedBlockMeta getBlockMetaOfPage(PageId pageId) { - long blockId = BlockPageId.downcast(pageId).getBlockId(); - PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); - if (blockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - return blockMeta; - } - @Override @GuardedBy("getLock().writeLock()") public void addTempPage(PageId pageId, PageInfo pageInfo) { @@ -249,34 +236,19 @@ public void commitFile(String fileId, String newFileId) throws PageNotFoundExcep mDelegate.commitFile(fileId, newFileId); } - /** - * @param blockId - * @return the permanent block meta after committing - */ - public PagedBlockMeta commit(long blockId) { - PagedTempBlockMeta tempBlockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); - if (tempBlockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - PagedBlockMeta blockMeta = new PagedBlockMeta(tempBlockMeta.getBlockId(), - tempBlockMeta.getBlockSize(), tempBlockMeta.getDir()); - mTempBlocks.remove(tempBlockMeta); - mBlocks.add(blockMeta); - try { - commitFile(BlockPageId.tempFileIdOf(blockId), - BlockPageId.fileIdOf(blockId, blockMeta.getBlockSize())); - } catch (PageNotFoundException e) { - // this should be unreachable, since we have checked the existence of the block - // otherwise it's a bug - LOG.error("Cannot commit block {} as no pages are found", blockId, e); - throw new RuntimeException(e); - } - return blockMeta; + @Override + public PageStoreDir getStoreDirOfFile(String fileId) throws FileDoesNotExistException { + return null; } @Override - public List getStoreDirs() { - return mDelegate.getStoreDirs(); + public Optional getUsage() { + return Optional.empty(); + } + + @Override + public PageInfo removePage(PageId pageId, boolean isTemporary) throws PageNotFoundException { + return null; } @Override @@ -298,6 +270,11 @@ public PageInfo removePage(PageId pageId) throws PageNotFoundException { return pageInfo; } + @Override + public List getStoreDirs() { + return mDelegate.getStoreDirs(); + } + @Override @GuardedBy("getLock().readLock()") public long bytes() { @@ -317,6 +294,11 @@ public void reset() { mBlocks.clear(); } + @Override + public Set getAllPagesByFileId(String fileId) { + return null; + } + @Override @GuardedBy("getLock().readLock()") public PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir) { @@ -358,6 +340,31 @@ public void registerBlockStoreEventListener(BlockStoreEventListener listener) { mBlockStoreEventListeners.add(listener); } + /** + * @param blockId + * @return the permanent block meta after committing + */ + public PagedBlockMeta commit(long blockId) { + PagedTempBlockMeta tempBlockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); + if (tempBlockMeta == null) { + throw new BlockDoesNotExistRuntimeException(blockId); + } + PagedBlockMeta blockMeta = new PagedBlockMeta(tempBlockMeta.getBlockId(), + tempBlockMeta.getBlockSize(), tempBlockMeta.getDir()); + mTempBlocks.remove(tempBlockMeta); + mBlocks.add(blockMeta); + try { + commitFile(BlockPageId.tempFileIdOf(blockId), + BlockPageId.fileIdOf(blockId, blockMeta.getBlockSize())); + } catch (PageNotFoundException e) { + // this should be unreachable, since we have checked the existence of the block + // otherwise it's a bug + LOG.error("Cannot commit block {} as no pages are found", blockId, e); + throw new RuntimeException(e); + } + return blockMeta; + } + /** * @return brief store meta */ @@ -412,4 +419,19 @@ private static PagedBlockStoreDir downcast(PageStoreDir pageStoreDir) { String.format("Unexpected page store dir type %s, for worker page store it should be %s", pageStoreDir.getClass().getSimpleName(), PagedBlockStoreDir.class.getSimpleName())); } + + /** + * Gets the block meta for a page of the block. + * @param pageId the page ID + * @return block meta + * @throws BlockDoesNotExistRuntimeException when the block is not being stored in the store + */ + private PagedBlockMeta getBlockMetaOfPage(PageId pageId) { + long blockId = BlockPageId.downcast(pageId).getBlockId(); + PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); + if (blockMeta == null) { + throw new BlockDoesNotExistRuntimeException(blockId); + } + return blockMeta; + } } diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java index 41b7a95d4016..69c12528caa6 100644 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java +++ b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java @@ -14,8 +14,9 @@ import alluxio.client.file.CacheContext; import alluxio.client.file.cache.CacheManager; import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.exception.runtime.AlluxioRuntimeException; +import alluxio.file.NettyBufTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.grpc.ErrorType; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; @@ -105,7 +106,7 @@ private long read(ByteBuf byteBuf, long offset, long length) throws IOException Preconditions.checkArgument(byteBuf.writableBytes() >= length, "buffer overflow, trying to write %s bytes, only %s writable", length, byteBuf.writableBytes()); - PageReadTargetBuffer target = new NettyBufTargetBuffer(byteBuf); + ReadTargetBuffer target = new NettyBufTargetBuffer(byteBuf); long bytesRead = 0; while (bytesRead < length) { long pos = offset + bytesRead; diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java index b5d232651200..e26a683a6be0 100644 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java +++ b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java @@ -14,6 +14,7 @@ import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_MEDIUM; import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_TIER; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; @@ -171,6 +172,10 @@ public boolean reserve(long bytes) { return mDelegate.reserve(bytes); } + @Override + public void deleteTempPage(PageInfo bytes) { + } + @Override public long deletePage(PageInfo pageInfo) { long blockId = BlockPageId.downcast(pageInfo.getPageId()).getBlockId(); @@ -277,4 +282,9 @@ public Set getBlockPages(long blockId) { return mBlockToPagesMap.get(blockId).stream().map(PageInfo::getPageId) .collect(Collectors.toSet()); } + + @Override + public Optional getUsage() { + return Optional.empty(); + } } diff --git a/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java b/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java index 2c21647ec766..b7aa85325f9b 100644 --- a/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java +++ b/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java @@ -13,12 +13,16 @@ import alluxio.client.file.CacheContext; import alluxio.client.file.cache.CacheManager; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; /** * Implementation of cache manager that stores cached data in byte arrays in memory. @@ -36,6 +40,11 @@ class ByteArrayCacheManager implements CacheManager { mPages = new HashMap<>(); } + @Override + public void commitFile(String fileId) { + throw new UnsupportedOperationException("commitFile method is unsupported. "); + } + @Override public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { byte[] data = new byte[page.remaining()]; @@ -46,7 +55,7 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { if (!mPages.containsKey(pageId)) { return 0; @@ -56,6 +65,24 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return bytesToRead; } + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + return bytesRead; + } + byte[] page = externalDataSupplier.get(); + if (page.length == 0) { + return 0; + } + buffer.writeBytes(page, pageOffset, bytesToRead); + put(pageId, page, cacheContext); + return bytesToRead; + } + @Override public boolean delete(PageId pageId) { return mPages.remove(pageId) != null; @@ -72,7 +99,50 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac } @Override - public void close() throws Exception { + public void deleteFile(String fileId) { + mPages.keySet().removeIf(pageId -> pageId.getFileId().equals(fileId)); + } + + @Override + public void deleteTempFile(String fileId) { + mPages.keySet().removeIf(pageId -> pageId.getFileId().equals(fileId)); + } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + return Optional.empty(); + } + + class Usage implements CacheUsage { + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + + @Override + public long used() { + return mPages.values().stream().mapToInt(page -> page.length).sum(); + } + + @Override + public long available() { + return Integer.MAX_VALUE; + } + + @Override + public long capacity() { + return Integer.MAX_VALUE; + } + } + + @Override + public void close() { // no-op } } diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java index fbbc687a5105..f13ba92abbfb 100644 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java +++ b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java @@ -25,12 +25,12 @@ import alluxio.client.file.cache.PageStore; import alluxio.client.file.cache.evictor.CacheEvictor; import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; import alluxio.client.file.cache.store.LocalPageStoreDir; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; +import alluxio.file.ByteArrayTargetBuffer; import alluxio.util.CommonUtils; import alluxio.util.WaitForOptions; import alluxio.util.io.BufferUtils; diff --git a/pom.xml b/pom.xml index 9a42a65aaffa..f5caf2b4ab5c 100644 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,7 @@ 1.6.3 5.3.5 2.0.7 + 1.0.0 0.8.0 31.0.1-jre 1.11.0 @@ -406,6 +407,21 @@ netty-transport-native-unix-common ${netty.version}     + + io.prometheus + prometheus-metrics-core + ${prometheus.client.version} + + + io.prometheus + prometheus-metrics-exporter-servlet-jakarta + ${prometheus.client.version} + + + io.prometheus + prometheus-metrics-instrumentation-jvm + ${prometheus.client.version} + io.prometheus simpleclient @@ -426,6 +442,11 @@ swagger-annotations 1.6.2 + + jakarta.servlet + jakarta.servlet-api + 5.0.0 + jakarta.ws.rs jakarta.ws.rs-api diff --git a/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java b/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java index 49ac00cd88b3..3149a7bdf979 100644 --- a/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java +++ b/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import alluxio.Constants; @@ -84,12 +85,6 @@ public void after() throws Exception { } } - @Test - public void newCacheRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testNewCache(); - } - @Test public void newCacheLocal() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -111,12 +106,6 @@ private void testPageCached(PageId pageId) { assertArrayEquals(PAGE, mBuffer); } - @Test - public void loadCacheRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testLoadCache(); - } - @Test public void loadCacheLocal() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -172,25 +161,6 @@ public void loadCacheMismatchedPageSize() throws Exception { testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_PAGE_SIZE, PAGE_SIZE_BYTES * 2); } - @Test - public void loadCacheMismatchedStoreTypeRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); - testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - } - - @Test - public void loadCacheMismatchedStoreTypeLocal() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); - } - - @Test - public void loadCacheSmallerNewCacheSizeRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_SIZE, - String.valueOf(CACHE_SIZE_BYTES / 2)); - } - @Test public void loadCacheSmallerNewCacheSizeLocal() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -215,6 +185,13 @@ public void loadCacheSmallerNewCacheSizeLocal() throws Exception { } } + /** + * Test the case that cache manager will properly handle invalid page file. + * if there is an invalid page file in the cache dir, cache manage will not recognize such + * file and will delete this file. Other valid page files are not affected + * and the cache manager should be able to read data from them normally. + * @throws Exception + */ @Test public void loadCacheWithInvalidPageFile() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -222,9 +199,14 @@ public void loadCacheWithInvalidPageFile() throws Exception { mCacheManager.close(); // creates with an invalid page file stored String rootDir = mPageMetaStore.getStoreDirs().get(0).getRootPath().toString(); - FileUtils.createFile(Paths.get(rootDir, "invalidPageFile").toString()); + String invalidPageFileName = Paths.get(rootDir, "invalidPageFile").toString(); + FileUtils.createFile(invalidPageFileName); mCacheManager = LocalCacheManager.create(mCacheManagerOptions, mPageMetaStore); - assertEquals(0, mCacheManager.get(PAGE_ID, PAGE_SIZE_BYTES, mBuffer, 0)); + // There is an invalid file in the cache dir. But the cache manager will not recognize it as a + // valid page file and will delete it, and then will continue starting as normal. + assertEquals(PAGE_SIZE_BYTES, mCacheManager.get(PAGE_ID, PAGE_SIZE_BYTES, mBuffer, 0)); + assertArrayEquals(PAGE, mBuffer); + assertFalse(FileUtils.exists(invalidPageFileName)); } @Test