diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java index 2ca67855a047..1269d0e68338 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManager.java @@ -12,12 +12,15 @@ package alluxio.client.file.cache; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.PropertyKey; +import alluxio.exception.PageNotFoundException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.metrics.MultiDimensionalMetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.resource.LockResource; import com.codahale.metrics.Counter; @@ -27,16 +30,19 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; +import java.util.function.Supplier; import javax.annotation.concurrent.GuardedBy; /** * Interface for managing cached pages. */ -public interface CacheManager extends AutoCloseable { +public interface CacheManager extends AutoCloseable, CacheStatus { + Logger LOG = LoggerFactory.getLogger(CacheManager.class); /** * State of a cache. @@ -119,6 +125,12 @@ public static CacheManager create(AlluxioConfiguration conf, try { boolean isShadowCacheEnabled = conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_SHADOW_ENABLED); + boolean isNettyDataTransmissionEnable = false; + // Note that Netty data transmission doesn't support async write + if (isNettyDataTransmissionEnable) { + options.setIsAsyncWriteEnabled(false); + } + MultiDimensionalMetricsSystem.setCacheStorageSupplier(pageMetaStore::bytes); if (isShadowCacheEnabled) { return new NoExceptionCacheManager( new CacheManagerWithShadowCache(LocalCacheManager.create(options, pageMetaStore), @@ -140,6 +152,8 @@ static void clear() { CacheManager manager = CACHE_MANAGER.getAndSet(null); if (manager != null) { manager.close(); + MultiDimensionalMetricsSystem.setCacheStorageSupplier( + MultiDimensionalMetricsSystem.NULL_SUPPLIER); } } catch (Exception e) { LOG.warn("Failed to close CacheManager: {}", e.toString()); @@ -236,6 +250,20 @@ default int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, return get(pageId, pageOffset, bytesToRead, buffer, offsetInBuffer, CacheContext.defaults()); } + /** + * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. + * + * @param pageId page identifier + * @param pageOffset offset into the page + * @param buffer destination buffer to write + * @param cacheContext cache related context + * @return number of bytes read, 0 if page is not found, -1 on errors + */ + default int get(PageId pageId, int pageOffset, ReadTargetBuffer buffer, + CacheContext cacheContext) { + throw new UnsupportedOperationException("This method is unsupported. "); + } + /** * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. * @@ -263,9 +291,24 @@ default int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, i * @param cacheContext cache related context * @return number of bytes read, 0 if page is not found, -1 on errors */ - int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, CacheContext cacheContext); + /** + * Reads a part of a page if the queried page is found in the cache, stores the result in buffer. + * Loads the page otherwise. + * + * @param pageId page identifier + * @param pageOffset offset into the page + * @param bytesToRead number of bytes to read in this page + * @param buffer destination buffer to write + * @param cacheContext cache related context + * @param externalDataSupplier the external data supplier to read a page + * @return number of bytes read, 0 if page is not found, -1 on errors + */ + int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, Supplier externalDataSupplier); + /** * Get page ids by the given file id. * @param fileId file identifier @@ -276,6 +319,20 @@ default List getCachedPageIdsByFileId(String fileId, long fileLength) { throw new UnsupportedOperationException(); } + /** + * Deletes all pages of the given file. + * + * @param fileId the file id of the target file + */ + void deleteFile(String fileId); + + /** + * Deletes all temporary pages of the given file. + * + * @param fileId the file id of the target file + */ + void deleteTempFile(String fileId); + /** * Deletes a page from the cache. * @@ -289,6 +346,14 @@ default List getCachedPageIdsByFileId(String fileId, long fileLength) { */ State state(); + /** + * @param pageId the page id + * @return true if the page is cached. This method is not thread-safe as no lock is acquired + */ + default boolean hasPageUnsafe(PageId pageId) { + throw new UnsupportedOperationException(); + } + /** * * @param pageId @@ -306,4 +371,25 @@ default List getCachedPageIdsByFileId(String fileId, long fileLength) { default void invalidate(Predicate predicate) { throw new UnsupportedOperationException(); } + + @Override + Optional getUsage(); + + /** + * Commit the File. + * @param fileId the file ID + */ + void commitFile(String fileId); + + /** + * Get a {@link DataFileChannel} which wraps a {@link io.netty.channel.FileRegion}. + * @param pageId the page id + * @param pageOffset the offset inside the page + * @param bytesToRead the bytes to read + * @param cacheContext the cache context + * @return an object of {@link DataFileChannel} + */ + Optional getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, CacheContext cacheContext) + throws PageNotFoundException; } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java index e6c98b2306f4..d2039423ad03 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheManagerWithShadowCache.java @@ -14,11 +14,13 @@ import static alluxio.client.file.CacheContext.StatsUnit.BYTE; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.quota.CacheScope; import alluxio.conf.AlluxioConfiguration; +import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; @@ -26,6 +28,8 @@ import com.google.common.hash.PrimitiveSink; import java.nio.ByteBuffer; +import java.util.Optional; +import java.util.function.Supplier; import javax.annotation.Nonnull; /** @@ -46,6 +50,11 @@ public CacheManagerWithShadowCache(CacheManager cacheManager, AlluxioConfigurati mShadowCacheManager = ShadowCacheManager.create(conf); } + @Override + public void commitFile(String fileId) { + mCacheManager.commitFile(fileId); + } + @Override public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { updateShadowCache(pageId, page.remaining(), cacheContext); @@ -53,8 +62,21 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { + getOrUpdateShadowCache(pageId, bytesToRead, cacheContext); + return mCacheManager.get(pageId, pageOffset, bytesToRead, target, cacheContext); + } + + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, Supplier externalDataSupplier) { + getOrUpdateShadowCache(pageId, bytesToRead, cacheContext); + return mCacheManager.getAndLoad(pageId, pageOffset, bytesToRead, + buffer, cacheContext, externalDataSupplier); + } + + private void getOrUpdateShadowCache(PageId pageId, int bytesToRead, CacheContext cacheContext) { int nread = mShadowCacheManager.get(pageId, bytesToRead, getCacheScope(cacheContext)); if (nread > 0) { Metrics.SHADOW_CACHE_PAGES_HIT.inc(); @@ -64,7 +86,6 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf } Metrics.SHADOW_CACHE_PAGES_READ.inc(); Metrics.SHADOW_CACHE_BYTES_READ.inc(bytesToRead); - return mCacheManager.get(pageId, pageOffset, bytesToRead, target, cacheContext); } /** @@ -142,6 +163,27 @@ public void close() throws Exception { mCacheManager.close(); } + @Override + public void deleteFile(String fileId) { + mCacheManager.deleteFile(fileId); + } + + @Override + public void deleteTempFile(String fileId) { + mCacheManager.deleteTempFile(fileId); + } + + @Override + public Optional getUsage() { + return mCacheManager.getUsage(); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) throws PageNotFoundException { + return mCacheManager.getDataFileChannel(pageId, pageOffset, bytesToRead, cacheContext); + } + /** * Decrease each item's clock and clean stale items. */ diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java new file mode 100644 index 000000000000..1f5b3c5ae2e8 --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheStatus.java @@ -0,0 +1,26 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache; + +import java.util.Optional; + +/** + * Mixin interface for various cache status info. + */ +public interface CacheStatus { + /** + * Gets cache usage. + * + * @return cache usage, or none if reporting usage info is not supported + */ + Optional getUsage(); +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java new file mode 100644 index 000000000000..29f1f72da15f --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsage.java @@ -0,0 +1,249 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache; + +import alluxio.client.quota.CacheScope; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; +import java.util.Optional; + +/** + * Cache usage. + *
+ * Granularity + * A usage object of this interface is associated with a certain granularity, either global, + * of a cache directory, of a application-defined scope, or of a particular file. Coarse-grained + * cache usage objects may be partitioned into a finer-grained one, for example the global usage + * can be partitioned into usages of each cache directory. + *
+ * Stats + * The following cache usage stats are reported for the granularity of the object: + *
    + *
  • Used: size of pages currently cached
    • + *
  • Available: size of free space, plus size of evictable pages
    • + *
  • Capacity: total capacity
    • + *
+ * Snapshot + * Cache usage object does not offer atomic view of their stats. Two subsequent calls to get the + * same stats may return different results, as the underlying cached contents may have changed + * between the calls. To get a snapshot of the cache stats, call {@link #snapshot()} to obtain + * an immutable view of the cache usage. + */ +//todo(bowen): allow introspection of the granularity of the cache usage object +public interface CacheUsage extends CacheUsageView { + /** + * Creates an immutable snapshot of the current cache usage. + * + * @return the snapshot + */ + default CacheUsageView snapshot() { + return new ImmutableCacheUsageView(used(), available(), capacity()); + } + + /** + * Gets a finer-grained view of cache usage. + *
Example of getting cache usage of a particular file: + *
+ * 
{@code
+   * cacheManager.getUsage()
+   *     .flatMap(usage -> usage.partitionedBy(PartitionDescriptor.file(fileId))
+   *     .map(CacheUsage::used)
+   *     .orElse(0)
+   * }
+ * + * @param partition how to partition the cache + * @return partitioned view of cache usage, none if this usage object does not support + * partitioning, or usage info for the specified partition is not found + */ + Optional partitionedBy(PartitionDescriptor partition); + + /** + * Partition descriptor. + * + * @param type of the identifier + */ + interface PartitionDescriptor { + /** + * Gets an identifier of the partition. + * + * @return identifier + */ + T getIdentifier(); + + /** + * Creates a partition for a specific file. + * + * @param fileId file ID + * @return the partition + */ + static FilePartition file(String fileId) { + return new FilePartition(fileId); + } + + /** + * Creates a partition for a directory. + * + * @param index dir index + * @return the partition + */ + static DirPartition dir(int index) { + return new DirPartition(index); + } + + /** + * Creates a partition of a cache scope. + * + * @param scope the cache scope + * @return the partition + */ + static ScopePartition scope(CacheScope scope) { + return new ScopePartition(scope); + } + } + + /** + * Partition on a particular cache directory. + */ + final class DirPartition implements PartitionDescriptor { + private final int mIndex; + + /** + * Creates a partition based on the directory's index. + * + * @param index dir index + */ + public DirPartition(int index) { + mIndex = index; + } + + @Override + public Integer getIdentifier() { + return mIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DirPartition that = (DirPartition) o; + return mIndex == that.mIndex; + } + + @Override + public int hashCode() { + return Objects.hash(mIndex); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("dirIndex", mIndex) + .toString(); + } + } + + /** + * Partition on a cache scope. + */ + final class ScopePartition implements PartitionDescriptor { + private final CacheScope mScope; + + /** + * Creates a partition over a cache scope. + * + * @param scope cache scope + */ + public ScopePartition(CacheScope scope) { + mScope = scope; + } + + @Override + public CacheScope getIdentifier() { + return mScope; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ScopePartition that = (ScopePartition) o; + return Objects.equals(mScope, that.mScope); + } + + @Override + public int hashCode() { + return Objects.hash(mScope); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("scope", mScope) + .toString(); + } + } + + /** + * Partition on a particular file. + */ + final class FilePartition implements PartitionDescriptor { + private final String mFileId; + + /** + * Creates a partition over a file ID. + * + * @param fileId the file ID + */ + public FilePartition(String fileId) { + mFileId = fileId; + } + + @Override + public String getIdentifier() { + return mFileId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + FilePartition that = (FilePartition) o; + return Objects.equals(mFileId, that.mFileId); + } + + @Override + public int hashCode() { + return Objects.hash(mFileId); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("fileId", mFileId) + .toString(); + } + } +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java new file mode 100644 index 000000000000..45c2ced3c5d2 --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/CacheUsageView.java @@ -0,0 +1,119 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache; + +import com.google.common.base.MoreObjects; + +import java.util.Objects; +import java.util.Optional; + +/** + * Immutable view of cache usage stats. + */ +public interface CacheUsageView { + /** + * Bytes currently used by the cache. + * This includes both evictable and inevitable pages. + * + * @return number of bytes being used + */ + long used(); + + /** + * Bytes that are available for caching. + * This includes free space, as well as the space occupied by evictable pages. + * + * @return number of bytes that can be used for caching + */ + long available(); + + /** + * Total capacity of the cache. + * @return total number of bytes the cache is allowed to use + */ + long capacity(); + + /** + * Immutable holder of cache stats. + */ + final class ImmutableCacheUsageView implements CacheUsage { + private final long mUsed; + private final long mAvailable; + private final long mCapacity; + + /** + * Constructor. + * + * @param used used + * @param available available + * @param capacity capacity + */ + public ImmutableCacheUsageView(long used, long available, long capacity) { + mUsed = used; + mAvailable = available; + mCapacity = capacity; + } + + @Override + public long used() { + return mUsed; + } + + @Override + public long available() { + return mAvailable; + } + + @Override + public long capacity() { + return mCapacity; + } + + @Override + public CacheUsageView snapshot() { + return this; + } + + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ImmutableCacheUsageView that = (ImmutableCacheUsageView) o; + return mUsed == that.mUsed + && mAvailable == that.mAvailable + && mCapacity == that.mCapacity; + } + + @Override + public int hashCode() { + return Objects.hash(mUsed, mAvailable, mCapacity); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("used", mUsed) + .add("available", mAvailable) + .add("capacity", mCapacity) + .toString(); + } + } +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java index 1c8e2b748119..daa12da40dc3 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/DefaultPageMetaStore.java @@ -20,6 +20,7 @@ import alluxio.client.quota.CacheScope; import alluxio.collections.IndexDefinition; import alluxio.collections.IndexedSet; +import alluxio.exception.FileDoesNotExistException; import alluxio.exception.PageNotFoundException; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; @@ -31,6 +32,7 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -121,15 +123,24 @@ public void commitFile(String fileId, String newFileId) throws PageNotFoundExcep throw new PageNotFoundException( String.format("No Pages found for file %s when committing", fileId)); } - for (PageInfo oldPage : pages) { - PageId newPageId = new PageId(newFileId, oldPage.getPageId().getPageIndex()); - PageInfo newPageInfo = new PageInfo(newPageId, oldPage.getPageSize(), oldPage.getScope(), - oldPage.getLocalCacheDir()); - mPages.remove(oldPage); + for (PageInfo oldPageInfo : pages) { + PageId newPageId = new PageId(newFileId, oldPageInfo.getPageId().getPageIndex()); + PageInfo newPageInfo = new PageInfo(newPageId, oldPageInfo.getPageSize(), + oldPageInfo.getScope(), oldPageInfo.getLocalCacheDir()); + mPages.remove(oldPageInfo); mPages.add(newPageInfo); } } + @Override + public PageStoreDir getStoreDirOfFile(String fileId) throws FileDoesNotExistException { + PageInfo pageInfo = mPages.getFirstByField(INDEX_FILE_ID, fileId); + if (pageInfo == null) { + throw new FileDoesNotExistException(String.format("File %s does not exist in cache", fileId)); + } + return pageInfo.getLocalCacheDir(); + } + @Override public List getStoreDirs() { return mDirs; @@ -153,7 +164,7 @@ public PageInfo getPageInfo(PageId pageId) throws PageNotFoundException { @Override @GuardedBy("getLock()") - public PageInfo removePage(PageId pageId) throws PageNotFoundException { + public PageInfo removePage(PageId pageId, boolean isTemporary) throws PageNotFoundException { if (!mPages.contains(INDEX_PAGE_ID, pageId)) { throw new PageNotFoundException(String.format("Page %s could not be found", pageId)); } @@ -162,10 +173,20 @@ public PageInfo removePage(PageId pageId) throws PageNotFoundException { mPages.remove(pageInfo); mBytes.addAndGet(-pageInfo.getPageSize()); Metrics.SPACE_USED.dec(pageInfo.getPageSize()); - pageInfo.getLocalCacheDir().deletePage(pageInfo); + if (isTemporary) { + pageInfo.getLocalCacheDir().deleteTempPage(pageInfo); + } else { + pageInfo.getLocalCacheDir().deletePage(pageInfo); + } return pageInfo; } + @Override + @GuardedBy("getLock()") + public PageInfo removePage(PageId pageId) throws PageNotFoundException { + return removePage(pageId, false); + } + @Override public long bytes() { return mBytes.get(); @@ -206,6 +227,56 @@ PageInfo evictInternal(CacheEvictor evictor) { return victimInfo; } + @Override + @GuardedBy("getLock().readLock()") + public Set getAllPagesByFileId(String fileId) { + Set pages = mPages.getByField(INDEX_FILE_ID, fileId); + return pages; + } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + class Usage implements CacheUsage { + + @Override + public long used() { + return bytes(); + } + + @Override + public long available() { + return capacity() - used(); + } + + @Override + public long capacity() { + return mDirs.stream().mapToLong(PageStoreDir::getCapacityBytes).sum(); + } + + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + if (partition instanceof FilePartition) { + String fileId = ((FilePartition) partition).getIdentifier(); + Set pages = mPages.getByField(INDEX_FILE_ID, fileId); + long used = pages.stream().mapToLong(PageInfo::getPageSize).sum(); + long capacity = capacity(); + long available = capacity - bytes(); + return Optional.of(new ImmutableCacheUsageView(used, available, capacity)); + } + if (partition instanceof DirPartition) { + int dirIndex = ((DirPartition) partition).getIdentifier(); + if (dirIndex < 0 || dirIndex >= mDirs.size()) { + return Optional.empty(); + } + return mDirs.get(dirIndex).getUsage(); + } + return Optional.empty(); + } + } + private static final class Metrics { // Note that only counter can be added here. // Both meter and timer need to be used inline diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java index fa78f002c55e..2ed8dae90a3f 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileInStream.java @@ -17,14 +17,16 @@ import alluxio.client.file.CacheContext; import alluxio.client.file.FileInStream; import alluxio.client.file.URIStatus; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.ByteBufferTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.client.file.cache.context.CachePerThreadContext; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.PropertyKey; import alluxio.exception.AlluxioException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.ByteBufferTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.metrics.MultiDimensionalMetricsSystem; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -36,6 +38,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Optional; import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.NotThreadSafe; @@ -53,20 +56,20 @@ public class LocalCacheFileInStream extends FileInStream { /** Local store to store pages. */ private final CacheManager mCacheManager; + private final boolean mIsDora; private final boolean mQuotaEnabled; - /** Cache related context of this file. */ - private final CacheContext mCacheContext; /** File info, fetched from external FS. */ private final URIStatus mStatus; private final FileInStreamOpener mExternalFileInStreamOpener; private final int mBufferSize; + private final boolean mFallbackEnabled; private byte[] mBuffer = null; private long mBufferStartOffset; private long mBufferEndOffset; /** Stream reading from the external file system, opened once. */ - private FileInStream mExternalFileInStream; + private Optional mExternalFileInStream; /** Current position of the stream, relative to the start of the file. */ private long mPosition = 0; private boolean mClosed = false; @@ -99,20 +102,19 @@ public static void registerMetrics() { * @param fileOpener open file in the external file system if a cache miss occurs * @param cacheManager local cache manager * @param conf configuration + * @param externalFileInStream */ public LocalCacheFileInStream(URIStatus status, FileInStreamOpener fileOpener, - CacheManager cacheManager, AlluxioConfiguration conf) { + CacheManager cacheManager, AlluxioConfiguration conf, + Optional externalFileInStream) { mPageSize = conf.getBytes(PropertyKey.USER_CLIENT_CACHE_PAGE_SIZE); mExternalFileInStreamOpener = fileOpener; + mExternalFileInStream = externalFileInStream; mCacheManager = cacheManager; mStatus = status; + mIsDora = false; // Currently quota is only supported when it is set by external systems in status context mQuotaEnabled = conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED); - if (mQuotaEnabled && status.getCacheContext() != null) { - mCacheContext = status.getCacheContext(); - } else { - mCacheContext = CacheContext.defaults(); - } Metrics.registerGauges(); mBufferSize = (int) conf.getBytes(PropertyKey.USER_CLIENT_CACHE_IN_STREAM_BUFFER_SIZE); @@ -120,6 +122,7 @@ public LocalCacheFileInStream(URIStatus status, FileInStreamOpener fileOpener, if (mBufferSize > 0) { mBuffer = new byte[mBufferSize]; } + mFallbackEnabled = conf.getBoolean(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED); } @Override @@ -138,7 +141,7 @@ public int read(ByteBuffer buffer, int offset, int length) throws IOException { return totalBytesRead; } - private int bufferedRead(PageReadTargetBuffer targetBuffer, int length, + private int bufferedRead(ReadTargetBuffer targetBuffer, int length, ReadType readType, long position, Stopwatch stopwatch) throws IOException { if (mBuffer == null) { //buffer is disabled, read data from local cache directly. return localCachedRead(targetBuffer, length, readType, position, stopwatch); @@ -168,32 +171,31 @@ private int bufferedRead(PageReadTargetBuffer targetBuffer, int length, return dataReadFromBuffer; } - private int localCachedRead(PageReadTargetBuffer bytesBuffer, int length, + private int localCachedRead(ReadTargetBuffer bytesBuffer, int length, ReadType readType, long position, Stopwatch stopwatch) throws IOException { long currentPage = position / mPageSize; PageId pageId; CacheContext cacheContext = mStatus.getCacheContext(); - if (cacheContext != null && cacheContext.getCacheIdentifier() != null) { + if (cacheContext == null) { + cacheContext = CacheContext.defaults(); + } + if (cacheContext.getCacheIdentifier() != null) { pageId = new PageId(cacheContext.getCacheIdentifier(), currentPage); } else { - pageId = new PageId(Long.toString(mStatus.getFileId()), currentPage); + // In Dora, the fileId is generated by Worker or by local client, which maybe is not unique. + // So we use the ufs path hash as its fileId. + String fileId = Long.toString(mStatus.getFileId()); + pageId = new PageId(fileId, currentPage); } int currentPageOffset = (int) (position % mPageSize); int bytesLeftInPage = (int) (mPageSize - currentPageOffset); int bytesToReadInPage = Math.min(bytesLeftInPage, length); stopwatch.reset().start(); int bytesRead = - mCacheManager.get(pageId, currentPageOffset, bytesToReadInPage, bytesBuffer, mCacheContext); + mCacheManager.get(pageId, currentPageOffset, bytesToReadInPage, bytesBuffer, cacheContext); stopwatch.stop(); if (bytesRead > 0) { - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesRead); - if (cacheContext != null) { - cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), BYTE, - bytesRead); - cacheContext.incrementCounter( - MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), NANO, - stopwatch.elapsed(TimeUnit.NANOSECONDS)); - } + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).inc(); return bytesRead; } // on local cache miss, read a complete page from external storage. This will always make @@ -206,6 +208,7 @@ private int localCachedRead(PageReadTargetBuffer bytesBuffer, int length, // cache misses MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) .mark(bytesToReadInPage); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).inc(); if (cacheContext != null) { cacheContext.incrementCounter( MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getMetricName(), BYTE, @@ -215,13 +218,13 @@ private int localCachedRead(PageReadTargetBuffer bytesBuffer, int length, stopwatch.elapsed(TimeUnit.NANOSECONDS) ); } - mCacheManager.put(pageId, page, mCacheContext); + mCacheManager.put(pageId, page, cacheContext); } return bytesToReadInPage; } // TODO(binfan): take ByteBuffer once CacheManager takes ByteBuffer to avoid extra mem copy - private int readInternal(PageReadTargetBuffer targetBuffer, int offset, int length, + private int readInternal(ReadTargetBuffer targetBuffer, int offset, int length, ReadType readType, long position, boolean isPositionedRead) throws IOException { Preconditions.checkArgument(length >= 0, "length should be non-negative"); Preconditions.checkArgument(offset >= 0, "offset should be non-negative"); @@ -284,8 +287,27 @@ public long remaining() { @Override public int positionedRead(long pos, byte[] b, int off, int len) throws IOException { - return readInternal(new ByteArrayTargetBuffer(b, off), off, len, ReadType.READ_INTO_BYTE_ARRAY, - pos, true); + if (!CachePerThreadContext.get().getCacheEnabled()) { + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) + .mark(len); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).inc(); + len = getExternalFileInStream().positionedRead(pos, b, off, len); + MultiDimensionalMetricsSystem.EXTERNAL_DATA_READ.inc(len); + return len; + } + try { + return readInternal(new ByteArrayTargetBuffer(b, off), off, len, + ReadType.READ_INTO_BYTE_ARRAY, pos, true); + } catch (IOException | RuntimeException e) { + LOG.warn("Failed to read from Alluxio's page cache.", e); + if (mFallbackEnabled) { + MetricsSystem.counter(MetricKey.CLIENT_CACHE_POSITION_READ_FALLBACK.getName()).inc(); + len = getExternalFileInStream().positionedRead(pos, b, off, len); + MultiDimensionalMetricsSystem.EXTERNAL_DATA_READ.inc(len); + return len; + } + throw e; + } } @Override @@ -311,9 +333,7 @@ public void seek(long pos) { @Override public void unbuffer() { - if (mExternalFileInStream != null) { - mExternalFileInStream.unbuffer(); - } + mExternalFileInStream.ifPresent((stream) -> stream.unbuffer()); } /** @@ -331,19 +351,26 @@ private void checkIfClosed() { * @param position position to set the external stream to */ private FileInStream getExternalFileInStream(long position) throws IOException { + FileInStream externalFileInStream = getExternalFileInStream(); + long pageStart = position - (position % mPageSize); + if (externalFileInStream.getPos() != pageStart) { + externalFileInStream.seek(pageStart); + } + return externalFileInStream; + } + + private FileInStream getExternalFileInStream() throws IOException { try { - if (mExternalFileInStream == null) { - mExternalFileInStream = mExternalFileInStreamOpener.open(mStatus); - mCloser.register(mExternalFileInStream); + if (!mExternalFileInStream.isPresent()) { + FileInStream externalFileInStream = mExternalFileInStreamOpener.open(mStatus); + mExternalFileInStream = Optional.of(externalFileInStream); + mCloser.register(externalFileInStream); + return externalFileInStream; } + return mExternalFileInStream.get(); } catch (AlluxioException e) { throw new IOException(e); } - long pageStart = position - (position % mPageSize); - if (mExternalFileInStream.getPos() != pageStart) { - mExternalFileInStream.seek(pageStart); - } - return mExternalFileInStream; } /** @@ -386,6 +413,7 @@ private synchronized byte[] readExternalPage(long position, ReadType readType) } totalBytesRead += bytesRead; } + MultiDimensionalMetricsSystem.EXTERNAL_DATA_READ.inc(totalBytesRead); // Bytes read from external, may be larger than requests due to reading complete pages MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).mark(totalBytesRead); if (totalBytesRead != pageSize) { diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java index ed541972ce6b..f5673242dbef 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheFileSystem.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Optional; /** * A FileSystem implementation with a local cache. @@ -70,6 +71,7 @@ public FileInStream openFile(URIStatus status, OpenFilePOptions options) return mDelegatedFileSystem.openFile(status, options); } return new LocalCacheFileInStream(status, - uriStatus -> mDelegatedFileSystem.openFile(status, options), mCacheManager, mConf); + uriStatus -> mDelegatedFileSystem.openFile(status, options), mCacheManager, mConf, + Optional.empty()); } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java index f3edb5db43bc..4874820581cc 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/LocalCacheManager.java @@ -11,6 +11,8 @@ package alluxio.client.file.cache; +import static alluxio.client.file.CacheContext.StatsUnit.BYTE; +import static alluxio.client.file.CacheContext.StatsUnit.NANO; import static alluxio.client.file.cache.CacheManager.State.NOT_IN_USE; import static alluxio.client.file.cache.CacheManager.State.READ_ONLY; import static alluxio.client.file.cache.CacheManager.State.READ_WRITE; @@ -18,18 +20,23 @@ import static java.util.concurrent.TimeUnit.SECONDS; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreDir; import alluxio.client.quota.CacheQuota; import alluxio.client.quota.CacheScope; import alluxio.collections.ConcurrentHashSet; import alluxio.collections.Pair; +import alluxio.exception.FileDoesNotExistException; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.metrics.MultiDimensionalMetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.resource.LockResource; +import alluxio.util.ThreadFactoryUtils; import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; @@ -43,6 +50,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; @@ -54,6 +62,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Predicate; +import java.util.function.Supplier; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; @@ -61,12 +70,12 @@ /** * A class to manage & serve cached pages. This class coordinates various components to respond for * thread-safety and enforce cache replacement policies. - * + *

* The idea of creating a client-side cache is followed after "Improving In-Memory File System * Reading Performance by Fine-Grained User-Space Cache Mechanisms" by Gu et al, which illustrates * performance benefits for various read workloads. This class also introduces paging as a caching * unit. - * + *

* Lock hierarchy in this class: All operations must follow this order to operate on pages: *

    *
  1. Acquire corresponding page lock
    2. @@ -83,29 +92,38 @@ public class LocalCacheManager implements CacheManager { private static final int LOCK_SIZE = 1024; private final long mCacheSize; - /** A readwrite lock pool to guard individual pages based on striping. */ + /** + * A readwrite lock pool to guard individual pages based on striping. + */ private final ReadWriteLock[] mPageLocks = new ReentrantReadWriteLock[LOCK_SIZE]; private final List mPageStoreDirs; @GuardedBy("PageMetaStore.getLock()") private final PageMetaStore mPageMetaStore; - /** Executor service for execute the init tasks. */ + /** + * Executor service for execute the init tasks. + */ private final Optional mInitService; - /** Executor service for execute the async cache tasks. */ + /** + * Executor service for execute the async cache tasks. + */ private final Optional mAsyncCacheExecutor; /** Executor service for execute the cache ttl check tasks. */ private final Optional mTtlEnforcerExecutor; private final ConcurrentHashSet mPendingRequests; - /** State of this cache. */ + /** + * State of this cache. + */ private final AtomicReference mState = new AtomicReference<>(); private final CacheManagerOptions mOptions; + private final Optional> mPagePredicate; /** - * @param options the options of local cache manager + * @param options the options of local cache manager * @param pageMetaStore the metadata store for local cache * @return an instance of {@link LocalCacheManager} */ public static LocalCacheManager create(CacheManagerOptions options, - PageMetaStore pageMetaStore) + PageMetaStore pageMetaStore) throws IOException { LocalCacheManager manager = new LocalCacheManager(options, pageMetaStore); List pageStoreDirs = pageMetaStore.getStoreDirs(); @@ -124,7 +142,7 @@ public static LocalCacheManager create(CacheManagerOptions options, } /** - * @param options the options of local cache manager + * @param options the options of local cache manager * @param pageMetaStore the meta store manages the metadata */ @VisibleForTesting @@ -139,34 +157,98 @@ public static LocalCacheManager create(CacheManagerOptions options, mPendingRequests = new ConcurrentHashSet<>(); mAsyncCacheExecutor = options.isAsyncWriteEnabled() - ? Optional.of( - new ThreadPoolExecutor(mOptions.getAsyncWriteThreads(), - mOptions.getAsyncWriteThreads(), 60, TimeUnit.SECONDS, - new SynchronousQueue<>(), new ThreadPoolExecutor.CallerRunsPolicy())) + ? Optional.of(new ThreadPoolExecutor(mOptions.getAsyncWriteThreads(), + mOptions.getAsyncWriteThreads(), 60, TimeUnit.SECONDS, + new SynchronousQueue<>(), ThreadFactoryUtils.build( + "alluxio-async-cache-executor", true), + new ThreadPoolExecutor.CallerRunsPolicy())) : Optional.empty(); mInitService = - options.isAsyncRestoreEnabled() ? Optional.of(Executors.newSingleThreadExecutor()) : + options.isAsyncRestoreEnabled() ? Optional.of(Executors.newSingleThreadExecutor( + ThreadFactoryUtils.build("alluxio-init-service", true))) : Optional.empty(); if (options.isTtlEnabled()) { - mTtlEnforcerExecutor = Optional.of(newScheduledThreadPool(1)); + Predicate ttl = pageInfo -> { + try { + return System.currentTimeMillis() - pageInfo.getCreatedTimestamp() + >= options.getTtlThresholdSeconds() * 1000; + } catch (Exception ex) { + // In case of any exception, do not invalidate the cache + return false; + } + }; + mPagePredicate = Optional.of(ttl); + mTtlEnforcerExecutor = Optional.of(newScheduledThreadPool(1, + ThreadFactoryUtils.build("alluxio-ttl-executor", true))); mTtlEnforcerExecutor.get().scheduleAtFixedRate(() -> - LocalCacheManager.this.invalidate(pageInfo -> { - try { - return System.currentTimeMillis() - pageInfo.getCreatedTimestamp() - >= options.getTtlThresholdSeconds() * 1000; - } catch (Exception ex) { - // In case of any exception, do not invalidate the cache - return false; - } - }), 0, options.getTtlCheckIntervalSeconds(), SECONDS); + LocalCacheManager.this.invalidate(ttl), 0, options.getTtlCheckIntervalSeconds(), SECONDS); } else { mTtlEnforcerExecutor = Optional.empty(); + mPagePredicate = Optional.empty(); } Metrics.registerGauges(mCacheSize, mPageMetaStore); mState.set(READ_ONLY); Metrics.STATE.inc(); } + @Override + public Optional getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, CacheContext cacheContext) + throws PageNotFoundException { + Preconditions.checkArgument(pageOffset <= mOptions.getPageSize(), + "Read exceeds page boundary: offset=%s size=%s", + pageOffset, mOptions.getPageSize()); + LOG.debug("get({},pageOffset={}) enters", pageId, pageOffset); + if (mState.get() == NOT_IN_USE) { + Metrics.GET_NOT_READY_ERRORS.inc(); + Metrics.GET_ERRORS.inc(); + throw new PageNotFoundException(String.format("Page %s could not be found", pageId)); + } + ReadWriteLock pageLock = getPageLock(pageId); + long startTime = System.nanoTime(); + try (LockResource r = new LockResource(pageLock.readLock())) { + PageInfo pageInfo; + try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { + pageInfo = mPageMetaStore.getPageInfo(pageId); //check if page exists and refresh LRU items + } catch (PageNotFoundException e) { + LOG.debug("getDataChannel({},pageOffset={}) fails due to page not found in metastore", + pageId, pageOffset); + return Optional.empty(); + } + + try { + DataFileChannel dataFileChannel = pageInfo.getLocalCacheDir().getPageStore() + .getDataFileChannel(pageInfo.getPageId(), pageOffset, bytesToRead, + cacheContext.isTemporary()); + MultiDimensionalMetricsSystem.CACHED_DATA_READ.inc(bytesToRead); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).inc(); + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesToRead); + cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), BYTE, + bytesToRead); + LOG.debug("getDataChannel({},pageOffset={}) exits", pageId, pageOffset); + return Optional.of(dataFileChannel); + } catch (PageNotFoundException e) { + LOG.debug("getDataChannel({},pageOffset={}) fails due to page file not found", + pageId, pageOffset); + Metrics.GET_ERRORS.inc(); + Metrics.GET_STORE_READ_ERRORS.inc(); + // something is wrong to read this page, let's remove it from meta store + try (LockResource r2 = new LockResource(mPageMetaStore.getLock().writeLock())) { + mPageMetaStore.removePage(pageId); + return Optional.empty(); + } catch (PageNotFoundException ex) { + // best effort to remove this page from meta store and ignore the exception + Metrics.CLEANUP_GET_ERRORS.inc(); + return Optional.empty(); + } + } + } finally { + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), NANO, + System.nanoTime() - startTime); + } + } + /** * @param pageId page identifier * @return the page lock id @@ -221,9 +303,9 @@ enum PutResult { */ @Nullable private CacheScope checkScopeToEvict(int pageSize, - PageStoreDir pageStoreDir, - CacheScope scope, CacheQuota quota, - boolean forcedToEvict) { + PageStoreDir pageStoreDir, + CacheScope scope, CacheQuota quota, + boolean forcedToEvict) { if (mOptions.isQuotaEnabled()) { // Check quota usage for each scope for (CacheScope currentScope = scope; currentScope != null; @@ -325,9 +407,32 @@ private boolean putInternal(PageId pageId, ByteBuffer page, CacheContext cacheCo return false; } + @Override + public void commitFile(String fileId) { + // TODO(JiamingMai): we still need to commit the data (not only the page metadata) + // call commit method of PageStoreDir + try { + PageStoreDir dir = mPageMetaStore.getStoreDirOfFile(fileId); + dir.commit(fileId, fileId); + } catch (FileDoesNotExistException notExistException) { + LOG.error(notExistException.getMessage()); + } catch (IllegalStateException illegalStateException) { + // ignore the commit exception + LOG.error(illegalStateException.getMessage()); + } catch (IOException ioException) { + // ignore the commit exception + LOG.error(ioException.getMessage()); + } + } + private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheContext, - boolean forcedToEvict) { + boolean forcedToEvict) { LOG.debug("putInternal({},{} bytes) enters", pageId, page.remaining()); + if (pageId.getPageIndex() > 0 && page.remaining() == 0) { + LOG.error("cannot put an empty page except for the first page." + + "pageId={}, isTemporary={}", pageId, cacheContext.isTemporary()); + return PutResult.OTHER; + } PageInfo victimPageInfo = null; CacheScope scopeToEvict; ReadWriteLock pageLock = getPageLock(pageId); @@ -385,7 +490,7 @@ private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheC Pair pageLockPair = getPageLockPair(pageId, victimPageInfo.getPageId()); try (LockResource r1 = new LockResource(pageLockPair.getFirst().writeLock()); - LockResource r2 = new LockResource(pageLockPair.getSecond().writeLock())) { + LockResource r2 = new LockResource(pageLockPair.getSecond().writeLock())) { // Excise a two-phase commit to evict victim and add new page: // phase1: remove victim and add new page in metastore in a critical section protected by // metalock. Evictor will be updated inside metastore. @@ -413,6 +518,7 @@ private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheC try { pageStoreDir.getPageStore().delete(victim); // Bytes evicted from the cache + MultiDimensionalMetricsSystem.CACHED_EVICTED_DATA.inc(victimPageInfo.getPageSize()); MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_EVICTED.getName()) .mark(victimPageInfo.getPageSize()); // Errors when adding pages @@ -456,7 +562,7 @@ private PutResult putAttempt(PageId pageId, ByteBuffer page, CacheContext cacheC } private void addPageToMetaStore(PageId pageId, ByteBuffer page, CacheContext cacheContext, - PageStoreDir pageStoreDir) { + PageStoreDir pageStoreDir) { PageInfo pageInfo = new PageInfo(pageId, page.remaining(), cacheContext.getCacheScope(), pageStoreDir); if (cacheContext.isTemporary()) { @@ -477,8 +583,26 @@ private void undoAddPage(PageId pageId) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, - CacheContext cacheContext) { + public int get(PageId pageId, int pageOffset, ReadTargetBuffer buffer, + CacheContext cacheContext) { + ReadWriteLock pageLock = getPageLock(pageId); + long pageSize = -1L; + try (LockResource r = new LockResource(pageLock.readLock())) { + PageInfo pageInfo; + try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { + pageInfo = mPageMetaStore.getPageInfo(pageId); //check if page exists and refresh LRU items + } catch (PageNotFoundException e) { + LOG.debug("get({},pageOffset={}) fails due to page not found", pageId, pageOffset); + return 0; + } + pageSize = pageInfo.getPageSize(); + } + return get(pageId, pageOffset, (int) pageSize, buffer, cacheContext); + } + + @Override + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, + CacheContext cacheContext) { Preconditions.checkArgument(pageOffset <= mOptions.getPageSize(), "Read exceeds page boundary: offset=%s size=%s", pageOffset, mOptions.getPageSize()); Preconditions.checkArgument(bytesToRead <= buffer.remaining(), @@ -491,6 +615,7 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return -1; } ReadWriteLock pageLock = getPageLock(pageId); + long startTime = System.nanoTime(); try (LockResource r = new LockResource(pageLock.readLock())) { PageInfo pageInfo; try (LockResource r2 = new LockResource(mPageMetaStore.getLock().readLock())) { @@ -513,14 +638,61 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf } return -1; } + MultiDimensionalMetricsSystem.CACHED_DATA_READ.inc(bytesRead); + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesRead); + cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), BYTE, + bytesRead); LOG.debug("get({},pageOffset={}) exits", pageId, pageOffset); return bytesRead; + } finally { + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), NANO, + System.nanoTime() - startTime); } } @Override - public boolean delete(PageId pageId) { - LOG.debug("delete({}) enters", pageId); + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, + CacheContext cacheContext, Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).inc(); + return bytesRead; + } + // on local cache miss, read a complete page from external storage. This will always make + // progress or throw an exception + // Note that we cannot synchronize on the new page, as this will cause deadlock due to + // incompatible lock order within putAttempt + // Not synchronizing may cause the same page to be read multiple times from UFS by two or more + // concurrent requests, + // but this is acceptable, and is expected to be rare as long as the number of striping locks + // is big enough + long startTime = System.nanoTime(); + byte[] page = externalDataSupplier.get(); + long timeElapse = System.nanoTime() - startTime; + buffer.writeBytes(page, pageOffset, bytesToRead); + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()) + .mark(bytesToRead); + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).inc(); + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getMetricName(), BYTE, + bytesToRead); + cacheContext.incrementCounter( + MetricKey.CLIENT_CACHE_PAGE_READ_EXTERNAL_TIME_NS.getMetricName(), NANO, + timeElapse); + put(pageId, page, cacheContext); + return bytesToRead; + } + + /** + * delete the specified page. + * + * @param pageId page identifier + * @param isTemporary whether is it temporary or not + * @return whether the page is deleted successfully or not + */ + public boolean delete(PageId pageId, boolean isTemporary) { if (mState.get() != READ_WRITE) { Metrics.DELETE_NOT_READY_ERRORS.inc(); Metrics.DELETE_ERRORS.inc(); @@ -531,15 +703,15 @@ public boolean delete(PageId pageId) { PageInfo pageInfo; try (LockResource r1 = new LockResource(mPageMetaStore.getLock().writeLock())) { try { - pageInfo = mPageMetaStore.removePage(pageId); + pageInfo = mPageMetaStore.removePage(pageId, isTemporary); } catch (PageNotFoundException e) { - LOG.error("Failed to delete page {} from metaStore ", pageId, e); + LOG.debug("Failed to delete page {} from metaStore ", pageId, e); Metrics.DELETE_NON_EXISTING_PAGE_ERRORS.inc(); Metrics.DELETE_ERRORS.inc(); return false; } } - boolean ok = deletePage(pageInfo); + boolean ok = deletePage(pageInfo, isTemporary); LOG.debug("delete({}) exits, success: {}", pageId, ok); if (!ok) { Metrics.DELETE_STORE_DELETE_ERRORS.inc(); @@ -549,6 +721,11 @@ public boolean delete(PageId pageId) { } } + @Override + public boolean delete(PageId pageId) { + return delete(pageId, false); + } + @Override public State state() { return mState.get(); @@ -563,8 +740,11 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac } if (appendAt > 0) { byte[] newPage = new byte[appendAt + page.length]; - get(pageId, 0, appendAt, new ByteArrayTargetBuffer(newPage, 0), cacheContext); - delete(pageId); + int readBytes = get(pageId, 0, appendAt, + new ByteArrayTargetBuffer(newPage, 0), cacheContext); + boolean success = delete(pageId, cacheContext.isTemporary()); + LOG.debug("delete pageId: {}, appendAt: {}, readBytes: {}, success: {}", + pageId, appendAt, readBytes, success); System.arraycopy(page, 0, newPage, appendAt, page.length); return put(pageId, newPage, cacheContext); } @@ -575,6 +755,7 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac * Restores a page store at the configured location, updating meta store accordingly. * If restore process fails, cleanup the location and create a new page store. * This method is synchronized to ensure only one thread can enter and operate. + * * @param pageStoreDirs */ private void restoreOrInit(List pageStoreDirs) throws IOException { @@ -600,15 +781,28 @@ private void restoreOrInit(List pageStoreDirs) throws IOException } private boolean restore(PageStoreDir pageStoreDir) { + long restoredPages = mPageMetaStore.numPages(); + long restoredBytes = mPageMetaStore.bytes(); + long discardPages = Metrics.PAGE_DISCARDED.getCount(); + long discardBytes = Metrics.BYTE_DISCARDED.getCount(); LOG.info("Restoring PageStoreDir ({})", pageStoreDir.getRootPath()); + if (!Files.exists(pageStoreDir.getRootPath())) { LOG.error("Failed to restore PageStore: Directory {} does not exist", pageStoreDir.getRootPath()); return false; } try { - pageStoreDir.scanPages(pageInfo -> { - addPageToDir(pageStoreDir, pageInfo.get()); + pageStoreDir.scanPages(optionalPageInfo -> { + if (optionalPageInfo.isPresent()) { + PageInfo pageInfo = optionalPageInfo.get(); + if (mPagePredicate.isPresent()) { + addPageBasedOnPredicate(pageStoreDir, pageInfo); + } + else { + addPageToDir(pageStoreDir, pageInfo); + } + } }); } catch (IOException | RuntimeException e) { LOG.error("Failed to restore PageStore", e); @@ -616,11 +810,40 @@ private boolean restore(PageStoreDir pageStoreDir) { } LOG.info("PageStore ({}) restored with {} pages ({} bytes), " + "discarded {} pages ({} bytes)", - pageStoreDir.getRootPath(), mPageMetaStore.numPages(), mPageMetaStore.bytes(), - Metrics.PAGE_DISCARDED.getCount(), Metrics.BYTE_DISCARDED); + pageStoreDir.getRootPath(), mPageMetaStore.numPages() - restoredPages, + mPageMetaStore.bytes() - restoredBytes, Metrics.PAGE_DISCARDED.getCount() - discardPages, + Metrics.BYTE_DISCARDED.getCount() - discardBytes); return true; } + private void addPageBasedOnPredicate(PageStoreDir pageStoreDir, PageInfo pageInfo) { + boolean tested = mPagePredicate.get().test(pageInfo); + if (!tested) { + addPageToDir(pageStoreDir, pageInfo); + MetricsSystem.histogram(MetricKey.CLIENT_CACHE_PAGES_AGES.getName()) + .update(System.currentTimeMillis() - pageInfo.getCreatedTimestamp()); + } + else { + // delete page directly and no metadata put into meta store + ReadWriteLock pageLock = getPageLock(pageInfo.getPageId()); + boolean isPageDeleted; + try (LockResource r = new LockResource(pageLock.writeLock())) { + isPageDeleted = deletePage(pageInfo, false); + } + if (isPageDeleted) { + MetricsSystem.meter(MetricKey.CLIENT_CACHE_PAGES_INVALIDATED.getName()).mark(); + } + else { + LOG.debug("Failed to delete old pages {} when restore", pageInfo.getPageId()); + Metrics.DELETE_STORE_DELETE_ERRORS.inc(); + Metrics.DELETE_ERRORS.inc(); + // still show the age of the old page to warn users + MetricsSystem.histogram(MetricKey.CLIENT_CACHE_PAGES_AGES.getName()) + .update(System.currentTimeMillis() - pageInfo.getCreatedTimestamp()); + } + } + } + private void addPageToDir(PageStoreDir pageStoreDir, PageInfo pageInfo) { PageId pageId = pageInfo.getPageId(); ReadWriteLock pageLock = getPageLock(pageId); @@ -661,16 +884,56 @@ public List getCachedPageIdsByFileId(String fileId, long fileLength) { return pageIds; } + @Override + public boolean hasPageUnsafe(PageId pageId) { + return mPageMetaStore.hasPage(pageId); + } + + @Override + public void deleteFile(String fileId) { + Set pages; + try (LockResource r = new LockResource(mPageMetaStore.getLock().readLock())) { + pages = mPageMetaStore.getAllPagesByFileId(fileId); + } + pages.forEach(page -> delete(page.getPageId())); + } + + @Override + public void deleteTempFile(String fileId) { + Set pages; + try (LockResource r = new LockResource(mPageMetaStore.getLock().readLock())) { + pages = mPageMetaStore.getAllPagesByFileId(fileId); + } + pages.forEach(page -> delete(page.getPageId(), true)); + } + @Override public void invalidate(Predicate predicate) { + if (mState.get() != READ_WRITE) { + Metrics.DELETE_NOT_READY_ERRORS.inc(); + Metrics.DELETE_ERRORS.inc(); + return; + } mPageStoreDirs.forEach(dir -> { try { - dir.scanPages(pageInfo -> { - if (pageInfo.isPresent() && predicate.test(pageInfo.get())) { - delete(pageInfo.get().getPageId()); + dir.scanPages(pageInfoOpt -> { + if (pageInfoOpt.isPresent()) { + PageInfo pageInfo = pageInfoOpt.get(); + boolean isPageDeleted = false; + if (predicate.test(pageInfo)) { + isPageDeleted = delete(pageInfo.getPageId()); + } + if (isPageDeleted) { + MetricsSystem.meter(MetricKey.CLIENT_CACHE_PAGES_INVALIDATED.getName()).mark(); + } + else { + MetricsSystem.histogram(MetricKey.CLIENT_CACHE_PAGES_AGES.getName()) + .update(System.currentTimeMillis() - pageInfo.getCreatedTimestamp()); + } } }); } catch (IOException e) { + LOG.error("IOException occurs in page scan", e); throw new RuntimeException(e); } }); @@ -678,7 +941,7 @@ public void invalidate(Predicate predicate) { @Override public void close() throws Exception { - for (PageStoreDir pageStoreDir: mPageStoreDirs) { + for (PageStoreDir pageStoreDir : mPageStoreDirs) { pageStoreDir.close(); } mPageMetaStore.reset(); @@ -694,18 +957,20 @@ public void close() throws Exception { * @param pageInfo page info * @return true if successful, false otherwise */ - private boolean deletePage(PageInfo pageInfo) { + private boolean deletePage(PageInfo pageInfo, boolean isTemporary) { try { - pageInfo.getLocalCacheDir().getPageStore().delete(pageInfo.getPageId()); + pageInfo.getLocalCacheDir().getPageStore().delete(pageInfo.getPageId(), isTemporary); } catch (IOException | PageNotFoundException e) { - LOG.error("Failed to delete page {} from pageStore", pageInfo.getPageId(), e); + LOG.error("Failed to delete page {} (isTemporary: {}) from pageStore.", + pageInfo.getPageId(), isTemporary, e); return false; } return true; } private int getPage(PageInfo pageInfo, int pageOffset, int bytesToRead, - PageReadTargetBuffer target, CacheContext cacheContext) { + ReadTargetBuffer target, CacheContext cacheContext) { + int originOffset = target.offset(); try { int ret = pageInfo.getLocalCacheDir().getPageStore() .get(pageInfo.getPageId(), pageOffset, bytesToRead, target, @@ -714,80 +979,168 @@ private int getPage(PageInfo pageInfo, int pageOffset, int bytesToRead, // data read from page store is inconsistent from the metastore LOG.error("Failed to read page {}: supposed to read {} bytes, {} bytes actually read", pageInfo.getPageId(), bytesToRead, ret); + target.offset(originOffset); //reset the offset + //best efforts to delete the corrupted file without acquire the write lock + deletePage(pageInfo, false); return -1; } + } catch (PageCorruptedException e) { + LOG.error("Data corrupted page {} from pageStore", pageInfo.getPageId(), e); + target.offset(originOffset); //reset the offset + //best efforts to delete the corrupted file without acquire the write lock + deletePage(pageInfo, false); + return -1; } catch (IOException | PageNotFoundException e) { LOG.debug("Failed to get existing page {} from pageStore", pageInfo.getPageId(), e); + target.offset(originOffset); //reset the offset return -1; } return bytesToRead; } + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + private final class Usage implements CacheUsage { + @Override + public long used() { + return mPageMetaStore.bytes(); + } + + @Override + public long available() { + return capacity() - used(); + } + + @Override + public long capacity() { + return mPageStoreDirs.stream().mapToLong(PageStoreDir::getCapacityBytes).sum(); + } + + @Override + public Optional partitionedBy(PartitionDescriptor partitionDescriptor) { + if (partitionDescriptor instanceof DirPartition) { + int dirIndex = ((DirPartition) partitionDescriptor).getIdentifier(); + if (dirIndex >= 0 && dirIndex < mPageStoreDirs.size()) { + return mPageStoreDirs.get(dirIndex).getUsage(); + } else { + return Optional.empty(); + } + } + return mPageMetaStore.getUsage() + .flatMap(usage -> usage.partitionedBy(partitionDescriptor)); + } + } + private static final class Metrics { // Note that only counter/guage can be added here. // Both meter and timer need to be used inline // because new meter and timer will be created after {@link MetricsSystem.resetAllMetrics()} - /** Total number of bytes discarded when restoring the page store. */ + /** + * Total number of bytes discarded when restoring the page store. + */ private static final Counter BYTE_DISCARDED = MetricsSystem.counter(MetricKey.CLIENT_CACHE_BYTES_DISCARDED.getName()); - /** Errors when cleaning up a failed get operation. */ + /** + * Errors when cleaning up a failed get operation. + */ private static final Counter CLEANUP_GET_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_CLEANUP_GET_ERRORS.getName()); - /** Errors when cleaning up a failed put operation. */ + /** + * Errors when cleaning up a failed put operation. + */ private static final Counter CLEANUP_PUT_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_CLEANUP_PUT_ERRORS.getName()); - /** Errors when deleting pages. */ + /** + * Errors when deleting pages. + */ private static final Counter DELETE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_ERRORS.getName()); - /** Errors when deleting pages due to absence. */ + /** + * Errors when deleting pages due to absence. + */ private static final Counter DELETE_NON_EXISTING_PAGE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_NON_EXISTING_PAGE_ERRORS.getName()); - /** Errors when cache is not ready to delete pages. */ + /** + * Errors when cache is not ready to delete pages. + */ private static final Counter DELETE_NOT_READY_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_NOT_READY_ERRORS.getName()); - /** Errors when deleting pages due to failed delete in page stores. */ + /** + * Errors when deleting pages due to failed delete in page stores. + */ private static final Counter DELETE_STORE_DELETE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_DELETE_FROM_STORE_ERRORS.getName()); - /** Errors when getting pages. */ + /** + * Errors when getting pages. + */ private static final Counter GET_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_ERRORS.getName()); - /** Errors when cache is not ready to get pages. */ + /** + * Errors when cache is not ready to get pages. + */ private static final Counter GET_NOT_READY_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_NOT_READY_ERRORS.getName()); - /** Errors when getting pages due to failed read from page stores. */ + /** + * Errors when getting pages due to failed read from page stores. + */ private static final Counter GET_STORE_READ_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_GET_STORE_READ_ERRORS.getName()); - /** Total number of pages discarded when restoring the page store. */ + /** + * Total number of pages discarded when restoring the page store. + */ private static final Counter PAGE_DISCARDED = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PAGES_DISCARDED.getName()); - /** Errors when adding pages. */ + /** + * Errors when adding pages. + */ private static final Counter PUT_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ERRORS.getName()); - /** Errors when adding pages due to failed injection to async write queue. */ + /** + * Errors when adding pages due to failed injection to async write queue. + */ private static final Counter PUT_ASYNC_REJECTION_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_ASYNC_REJECTION_ERRORS.getName()); - /** Errors when adding pages due to failed eviction. */ + /** + * Errors when adding pages due to failed eviction. + */ private static final Counter PUT_EVICTION_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_EVICTION_ERRORS.getName()); - /** Errors when adding pages due to benign racing eviction. */ + /** + * Errors when adding pages due to benign racing eviction. + */ private static final Counter PUT_BENIGN_RACING_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_BENIGN_RACING_ERRORS.getName()); - /** Errors when adding pages due to insufficient space made after eviction. */ + /** + * Errors when adding pages due to insufficient space made after eviction. + */ private static final Counter PUT_INSUFFICIENT_SPACE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_INSUFFICIENT_SPACE_ERRORS.getName()); - /** Errors when cache is not ready to add pages. */ + /** + * Errors when cache is not ready to add pages. + */ private static final Counter PUT_NOT_READY_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_NOT_READY_ERRORS.getName()); - /** Errors when adding pages due to failed deletes in page store. */ + /** + * Errors when adding pages due to failed deletes in page store. + */ private static final Counter PUT_STORE_DELETE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_DELETE_ERRORS.getName()); - /** Errors when adding pages due to failed writes to page store. */ + /** + * Errors when adding pages due to failed writes to page store. + */ private static final Counter PUT_STORE_WRITE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_WRITE_ERRORS.getName()); - /** Errors when adding pages due to failed writes but before reaching cache capacity. */ + /** + * Errors when adding pages due to failed writes but before reaching cache capacity. + */ private static final Counter PUT_STORE_WRITE_NO_SPACE_ERRORS = MetricsSystem.counter(MetricKey.CLIENT_CACHE_PUT_STORE_WRITE_NO_SPACE_ERRORS.getName()); - /** State of the cache. */ + /** + * State of the cache. + */ private static final Counter STATE = MetricsSystem.counter(MetricKey.CLIENT_CACHE_STATE.getName()); diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java b/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java index 4438b496fd61..7e30bbe68abd 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/NoExceptionCacheManager.java @@ -12,9 +12,11 @@ package alluxio.client.file.cache; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.codahale.metrics.Counter; import org.slf4j.Logger; @@ -22,6 +24,8 @@ import java.nio.ByteBuffer; import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; /** * A wrapper class of CacheManager without throwing unchecked exceptions. @@ -38,6 +42,15 @@ public NoExceptionCacheManager(CacheManager cacheManager) { mCacheManager = cacheManager; } + @Override + public void commitFile(String fileId) { + try { + mCacheManager.commitFile(fileId); + } catch (Exception e) { + LOG.error("Failed to commit file {}", fileId, e); + } + } + @Override public boolean put(PageId pageId, byte[] page) { try { @@ -60,6 +73,25 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } } + @Override + public int get(PageId pageId, int pageOffset, ReadTargetBuffer buffer, + CacheContext cacheContext) { + int originalOffset = buffer.offset(); + try { + int bytesRead = mCacheManager.get(pageId, pageOffset, buffer, cacheContext); + if (bytesRead == -1) { + buffer.offset(originalOffset); + } + return bytesRead; + } catch (Exception e) { + LOG.error("Failed to get page {}", pageId, e); + Metrics.GET_ERRORS.inc(); + //In case any error in cache manager, revert the offset change in the buffer + buffer.offset(originalOffset); + return -1; + } + } + @Override public int get(PageId pageId, int bytesToRead, byte[] buffer, int offsetInBuffer) { try { @@ -98,7 +130,7 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, byte[] buffer, } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, CacheContext cacheContext) { try { return mCacheManager @@ -111,6 +143,21 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf } } + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + try { + return mCacheManager.getAndLoad(pageId, pageOffset, bytesToRead, + buffer, cacheContext, externalDataSupplier); + } catch (Exception e) { + LOG.error("Failed to get and load page {}, offset {} cacheContext {}", pageId, pageOffset, + cacheContext, e); + Metrics.GET_ERRORS.inc(); + return -1; + } + } + @Override public boolean delete(PageId pageId) { try { @@ -122,6 +169,23 @@ public boolean delete(PageId pageId) { } } + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + try { + return mCacheManager.getDataFileChannel(pageId, pageOffset, bytesToRead, cacheContext); + } catch (Exception e) { + if (e instanceof PageNotFoundException) { + // In cold read, this may be expected behavior + LOG.debug("Failed to getDataFileChannel of page {}", pageId, e); + } else { + LOG.error("Failed to getDataFileChannel of page {}", pageId, e); + } + Metrics.GET_ERRORS.inc(); + return Optional.empty(); + } + } + @Override public State state() { return mCacheManager.state(); @@ -146,6 +210,34 @@ public List getCachedPageIdsByFileId(String fileId, long fileLength) { return mCacheManager.getCachedPageIdsByFileId(fileId, fileLength); } + @Override + public boolean hasPageUnsafe(PageId pageId) { + return mCacheManager.hasPageUnsafe(pageId); + } + + @Override + public void deleteFile(String fileId) { + try { + mCacheManager.deleteFile(fileId); + } catch (Exception e) { + LOG.error("Failed to deleteFile for {}", fileId, e); + } + } + + @Override + public void deleteTempFile(String fileId) { + try { + mCacheManager.deleteTempFile(fileId); + } catch (Exception e) { + LOG.error("Failed to deleteFile for {}", fileId, e); + } + } + + @Override + public Optional getUsage() { + return mCacheManager.getUsage(); + } + private static final class Metrics { // Note that only counter/guage can be added here. // Both meter and timer need to be used inline diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java index 5bbaf90b5a9a..41ab24eaac0e 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/PageInfo.java @@ -125,6 +125,7 @@ public String toString() { .add("PageId", mPageId) .add("PageSize", mPageSize) .add("Scope", mCacheScope) + .add("CreateTimeMs", mCreatedTimestamp) .toString(); } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java index 673955559fac..c918c64deb13 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/PageMetaStore.java @@ -13,16 +13,18 @@ import alluxio.client.file.cache.store.PageStoreDir; import alluxio.client.quota.CacheScope; +import alluxio.exception.FileDoesNotExistException; import alluxio.exception.PageNotFoundException; import java.io.IOException; import java.util.List; +import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; /** * The metadata store for pages stored in cache. */ -public interface PageMetaStore { +public interface PageMetaStore extends CacheStatus { /** * @param options the options of cache @@ -71,6 +73,17 @@ static PageMetaStore create(CacheManagerOptions options) throws IOException { */ void commitFile(String fileId, String newFileId) throws PageNotFoundException; + /** + * Gets the store dir which the specified file is assigned to be cached in. + * Implementations should ensure that all pages + * of the same file are cached in the same directory, until all the pages of the file are evicted. + * + * @param fileId the file ID + * @return the store dir which caches the pages of the file + * @throws FileDoesNotExistException if the file is not being cached + */ + PageStoreDir getStoreDirOfFile(String fileId) throws FileDoesNotExistException; + /** * Gets the storage directories. * @@ -91,6 +104,15 @@ static PageMetaStore create(CacheManagerOptions options) throws IOException { */ PageInfo getPageInfo(PageId pageId) throws PageNotFoundException; + /** + * Removes a page. + * + * @param pageId page identifier + * @param isTemporary whether is it temporary page or not + * @return page info removed + */ + PageInfo removePage(PageId pageId, boolean isTemporary) throws PageNotFoundException; + /** * Removes a page. * @@ -114,6 +136,13 @@ static PageMetaStore create(CacheManagerOptions options) throws IOException { */ void reset(); + /** + * Gets all pages by the specified File ID. + * @param fileId the target file id + * @return a set of PageInfo's of this file, otherwise an empty set if not found + */ + Set getAllPagesByFileId(String fileId); + /** * @param pageStoreDir * @return a page to evict diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java index 660207295154..a73cd96a8939 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/PageStore.java @@ -11,15 +11,17 @@ package alluxio.client.file.cache; +import alluxio.Constants; import alluxio.client.file.cache.store.LocalPageStore; import alluxio.client.file.cache.store.MemoryPageStore; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; -import alluxio.client.file.cache.store.RocksPageStore; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; +import alluxio.util.logging.SamplingLogger; import com.codahale.metrics.Counter; import org.slf4j.Logger; @@ -34,6 +36,7 @@ */ public interface PageStore extends AutoCloseable { Logger LOG = LoggerFactory.getLogger(PageStore.class); + Logger SAMPLING_LOG = new SamplingLogger(LOG, Constants.SECOND_MS * 10); /** * Create an instance of PageStore. @@ -48,9 +51,6 @@ static PageStore create(PageStoreOptions options) { case LOCAL: pageStore = new LocalPageStore(options); break; - case ROCKS: - pageStore = RocksPageStore.open(options); - break; case MEM: pageStore = new MemoryPageStore((int) options.getPageSize()); break; @@ -127,7 +127,7 @@ void put(PageId pageId, * @throws IOException when the store fails to read this page * @throws PageNotFoundException when the page isn't found in the store */ - default int get(PageId pageId, PageReadTargetBuffer buffer) + default int get(PageId pageId, ReadTargetBuffer buffer) throws IOException, PageNotFoundException { return get(pageId, 0, (int) buffer.remaining(), buffer, false); } @@ -144,7 +144,7 @@ default int get(PageId pageId, PageReadTargetBuffer buffer) * @throws PageNotFoundException when the page isn't found in the store * @throws IllegalArgumentException when the page offset exceeds the page size */ - default int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer) + default int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer) throws IOException, PageNotFoundException { return get(pageId, pageOffset, bytesToRead, buffer, false); } @@ -162,10 +162,23 @@ default int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBu * @throws PageNotFoundException when the page isn't found in the store * @throws IllegalArgumentException when the page offset exceeds the page size */ - int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, boolean isTemporary) throws IOException, PageNotFoundException; + /** + * Deletes a temporary page from the store. + * + * @param pageId page identifier + * @param isTemporary whether is to delete the temporary page or not + * @throws IOException when the store fails to delete this page + * @throws PageNotFoundException when the page isn't found in the store + */ + default void delete(PageId pageId, boolean isTemporary) + throws IOException, PageNotFoundException { + delete(pageId); + } + /** * Deletes a page from the store. * @@ -173,7 +186,9 @@ int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buf * @throws IOException when the store fails to delete this page * @throws PageNotFoundException when the page isn't found in the store */ - void delete(PageId pageId) throws IOException, PageNotFoundException; + default void delete(PageId pageId) throws IOException, PageNotFoundException { + delete(pageId, false); + } /** * Commit a temporary file. @@ -200,6 +215,21 @@ default void abort(String fileId) throws IOException { throw new UnsupportedOperationException(); } + /** + * Get a {@link DataFileChannel} which wraps a {@link io.netty.channel.FileRegion}. + * @param pageId the page id + * @param pageOffset the offset inside the page + * @param bytesToRead the bytes to read + * @param isTemporary whether it is temporary or not + * @return an object of {@link DataFileChannel} + * @throws PageNotFoundException + */ + default DataFileChannel getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, boolean isTemporary) + throws PageNotFoundException { + throw new UnsupportedOperationException(); + } + /** * Metrics. */ diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java index 0ccb31257dfa..6e406dff8eed 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/QuotaPageMetaStore.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -120,4 +121,23 @@ public PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir) { CacheEvictor evictor = mCacheEvictors.computeIfAbsent(cacheScope, k -> mSupplier.get()); return evictInternal(evictor); } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + class Usage extends DefaultPageMetaStore.Usage { + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + if (partition instanceof ScopePartition) { + CacheScope scope = ((ScopePartition) partition).getIdentifier(); + long capacity = capacity(); + long used = bytes(scope); + long available = capacity - bytes(); // not capacity - used! + return Optional.of(new ImmutableCacheUsageView(used, available, capacity)); + } + return super.partitionedBy(partition); + } + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java index d3339355d5b7..553708c56e4e 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/TimeBoundPageStore.java @@ -11,12 +11,13 @@ package alluxio.client.file.cache; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ReadTargetBuffer; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.codahale.metrics.Counter; import com.google.common.base.Preconditions; @@ -88,7 +89,7 @@ public void put(PageId pageId, } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Callable callable = () -> mPageStore.get(pageId, pageOffset, bytesToRead, target, isTemporary); @@ -136,6 +137,12 @@ public void delete(PageId pageId) throws IOException, PageNotFoundException { } } + @Override + public DataFileChannel getDataFileChannel(PageId pageId, int pageOffset, int bytesToRead, + boolean isTemporary) throws PageNotFoundException { + return mPageStore.getDataFileChannel(pageId, pageOffset, bytesToRead, isTemporary); + } + @Override public void close() throws Exception { mExecutorService.shutdown(); diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java b/core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java new file mode 100644 index 000000000000..12b81ca4ad17 --- /dev/null +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/context/CachePerThreadContext.java @@ -0,0 +1,43 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.client.file.cache.context; + +/** + * A per-thread cache context for local cache. + */ +public class CachePerThreadContext { + private static ThreadLocal sContext = + ThreadLocal.withInitial(() -> new CachePerThreadContext()); + + private boolean mCacheEnabled = true; + + /** + * @return per-thread cache context + */ + public static CachePerThreadContext get() { + return sContext.get(); + } + + /** + * @param isCacheEnabled + */ + public void setCacheEnabled(boolean isCacheEnabled) { + mCacheEnabled = isCacheEnabled; + } + + /** + * @return if cache is enabled + */ + public boolean getCacheEnabled() { + return mCacheEnabled; + } +} diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java index 446c9e5e5a95..b972d944a129 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStore.java @@ -15,13 +15,18 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.io.FileUtils; +import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.RandomAccessFile; @@ -65,38 +70,44 @@ public void put(PageId pageId, boolean isTemporary) throws ResourceExhaustedException, IOException { Path pagePath = getPagePath(pageId, isTemporary); try { + LOG.debug("Put page: {}, page's position: {}, page's limit: {}, page's capacity: {}", + pageId, page.position(), page.limit(), page.capacity()); if (!Files.exists(pagePath)) { Path parent = Preconditions.checkNotNull(pagePath.getParent(), "parent of cache file should not be null"); Files.createDirectories(parent); - Files.createFile(pagePath); } // extra try to ensure output stream is closed try (FileOutputStream fos = new FileOutputStream(pagePath.toFile(), false)) { fos.getChannel().write(page); } - } catch (Exception e) { + } catch (Throwable t) { Files.deleteIfExists(pagePath); - if (e.getMessage() != null && e.getMessage().contains(ERROR_NO_SPACE_LEFT)) { + if (t.getMessage() != null && t.getMessage().contains(ERROR_NO_SPACE_LEFT)) { throw new ResourceExhaustedException( - String.format("%s is full, configured with %d bytes", mRoot, mCapacity), e); + String.format("%s is full, configured with %d bytes", mRoot, mCapacity), t); } - throw new IOException("Failed to write file " + pagePath + " for page " + pageId, e); + throw new IOException("Failed to write file " + pagePath + " for page " + pageId, t); } } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); - Path pagePath = getPagePath(pageId, isTemporary); - if (!Files.exists(pagePath)) { - throw new PageNotFoundException(pagePath.toString()); + Preconditions.checkArgument(bytesToRead >= 0, "bytes to read should be non-negative"); + if (target.remaining() == 0 || bytesToRead == 0) { + return 0; } - long pageLength = pagePath.toFile().length(); - Preconditions.checkArgument(pageOffset <= pageLength, "page offset %s exceeded page size %s", - pageOffset, pageLength); + Path pagePath = getPagePath(pageId, isTemporary); try (RandomAccessFile localFile = new RandomAccessFile(pagePath.toString(), "r")) { + long pageLength = localFile.length(); + if (pageOffset + bytesToRead > pageLength) { + throw new PageCorruptedException(String.format( + "The page %s (%s) probably has been corrupted, " + + "page-offset %s, bytes to read %s, page file length %s", + pageId, pagePath, pageOffset, bytesToRead, pageLength)); + } int bytesSkipped = localFile.skipBytes(pageOffset); if (pageOffset != bytesSkipped) { throw new IOException( @@ -104,8 +115,7 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf pageId, pagePath, pageOffset, bytesSkipped)); } int bytesRead = 0; - int bytesLeft = (int) Math.min(pageLength - pageOffset, target.remaining()); - bytesLeft = Math.min(bytesLeft, bytesToRead); + int bytesLeft = Math.min((int) target.remaining(), bytesToRead); while (bytesLeft > 0) { int bytes = target.readFromFile(localFile, bytesLeft); if (bytes <= 0) { @@ -114,13 +124,27 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf bytesRead += bytes; bytesLeft -= bytes; } + if (bytesRead == 0) { + SAMPLING_LOG.warn("Read 0 bytes from page {}, the page is probably empty", pageId); + // no bytes have been read at all, but the requested length > 0 + // this means the file is empty + return -1; + } return bytesRead; + } catch (FileNotFoundException e) { + throw new PageNotFoundException(pagePath.toString()); } } - @Override - public void delete(PageId pageId) throws IOException, PageNotFoundException { - Path pagePath = getPagePath(pageId, false); + /** + * + * @param pageId page identifier + * @param isTemporary whether is to delete a temporary page or not + * @throws IOException + * @throws PageNotFoundException + */ + public void delete(PageId pageId, boolean isTemporary) throws IOException, PageNotFoundException { + Path pagePath = getPagePath(pageId, isTemporary); if (!Files.exists(pagePath)) { throw new PageNotFoundException(pagePath.toString()); } @@ -189,6 +213,44 @@ public Path getPagePath(PageId pageId, boolean isTemporary) { return filePath.resolve(Long.toString(pageId.getPageIndex())); } + @Override + public DataFileChannel getDataFileChannel( + PageId pageId, int pageOffset, int bytesToRead, boolean isTemporary) + throws PageNotFoundException { + Preconditions.checkArgument(pageOffset >= 0, + "page offset should be non-negative"); + Preconditions.checkArgument(!isTemporary, + "cannot acquire a data file channel to a temporary page"); + Path pagePath = getPagePath(pageId, isTemporary); + File pageFile = pagePath.toFile(); + if (!pageFile.exists()) { + throw new PageNotFoundException(pagePath.toString()); + } + + long fileLength = pageFile.length(); + if (fileLength == 0 && pageId.getPageIndex() > 0) { + // pages other than the first page should always be non-empty + // remove this malformed page + SAMPLING_LOG.warn("Length of page {} is 0, removing this malformed page", pageId); + try { + Files.deleteIfExists(pagePath); + } catch (IOException ignored) { + // do nothing + } + throw new PageNotFoundException(pagePath.toString()); + } + if (fileLength < pageOffset) { + throw new IllegalArgumentException( + String.format("offset %s exceeds length of page %s", pageOffset, fileLength)); + } + if (pageOffset + bytesToRead > fileLength) { + bytesToRead = (int) (fileLength - (long) pageOffset); + } + + DataFileChannel dataFileChannel = new DataFileChannel(pageFile, pageOffset, bytesToRead); + return dataFileChannel; + } + @Override public void close() { // no-op diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java index 3d13ba544313..c04d4e8da346 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/LocalPageStoreDir.java @@ -13,6 +13,7 @@ import static alluxio.client.file.cache.store.PageStoreDir.getFileBucket; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; @@ -100,7 +101,7 @@ public void scanPages(Consumer> pageInfoConsumer) throws IOEx /** * @param path path of a file - * @return the corresponding page info for the file otherwise null + * @return the corresponding page info for the file otherwise empty */ private Optional getPageInfo(Path path) { Optional pageId = getPageId(path); @@ -113,22 +114,47 @@ private Optional getPageInfo(Path path) { createdTime = creationTime.toMillis(); } catch (IOException e) { LOG.error("Failed to get file size for " + path, e); + deleteUnrecognizedPage(path); return Optional.empty(); } return Optional.of(new PageInfo(pageId.get(), pageSize, CacheScope.GLOBAL, this, createdTime)); } + deleteUnrecognizedPage(path); return Optional.empty(); } + /** + * Deletes an unrecognized page file due to various reasons. + * @param path the file path of a page file + */ + private void deleteUnrecognizedPage(Path path) { + Preconditions.checkState(path.startsWith(getRootPath()), + String.format("%s is not inside the cache dir (%s)!", path, getRootPath())); + try { + Files.delete(path); + } catch (IOException e) { + // ignore. + } + } + /** * @param path path of a file - * @return the corresponding page id, or null if the file name does not match the pattern + * @return the corresponding page id, or empty if the file name does not match the pattern */ private Optional getPageId(Path path) { Matcher matcher = mPagePattern.matcher(path.toString()); if (!matcher.matches()) { - LOG.error("Unrecognized page file " + path); + // @TODO(hua) define the mPagePattern and TEMP page Pattern as static class member to save + // CPU time and memory footprint. + if (Pattern.matches(String.format("%s/%d/%s/([^/]+)/(\\d+)", + Pattern.quote(mPageStoreOptions.getRootDir().toString()), + mPageStoreOptions.getPageSize(), LocalPageStore.TEMP_DIR), path.toString())) { + LOG.info("TEMP page file " + path + " is going to be deleted."); + } else { + LOG.error("Unrecognized page file " + path + "is going to be deleted."); + } + deleteUnrecognizedPage(path); return Optional.empty(); } try { @@ -136,6 +162,7 @@ private Optional getPageId(Path path) { String fileId = Preconditions.checkNotNull(matcher.group(2)); if (!fileBucket.equals(getFileBucket(mFileBuckets, fileId))) { LOG.error("Bucket number mismatch " + path); + deleteUnrecognizedPage(path); return Optional.empty(); } String fileName = Preconditions.checkNotNull(matcher.group(3)); @@ -143,7 +170,13 @@ private Optional getPageId(Path path) { return Optional.of(new PageId(fileId, pageIndex)); } catch (NumberFormatException e) { LOG.error("Illegal numbers in path " + path); + deleteUnrecognizedPage(path); return Optional.empty(); } } + + @Override + public Optional getUsage() { + return Optional.of(new QuotaManagedPageStoreDir.Usage()); + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java index 81dc14829c76..7fde058b6c27 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStore.java @@ -13,11 +13,14 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.LinkedList; @@ -57,7 +60,7 @@ public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOEx } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Preconditions.checkArgument(target != null, "buffer is null"); Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); @@ -66,8 +69,12 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf throw new PageNotFoundException(pageId.getFileId() + "_" + pageId.getPageIndex()); } MemPage page = mPageStoreMap.get(pageKey); - Preconditions.checkArgument(pageOffset <= page.getPageLength(), - "page offset %s exceeded page size %s", pageOffset, page.getPageLength()); + if (pageOffset + bytesToRead > page.getPageLength()) { + throw new PageCorruptedException(String.format( + "The page %s probably has been corrupted, " + + "page-offset %s, bytes to read %s, page file length %s", + pageId, pageOffset, bytesToRead, page.getPageLength())); + } int bytesLeft = (int) Math.min(page.getPageLength() - pageOffset, target.remaining()); bytesLeft = Math.min(bytesLeft, bytesToRead); target.writeBytes(page.getPage(), pageOffset, bytesLeft); @@ -84,6 +91,11 @@ public void delete(PageId pageId) throws IOException, PageNotFoundException { mPageStoreMap.remove(pageKey); } + @Override + public void commit(String fileId, String newFileId) throws IOException { + // noop because the pages are all in memory, there is no underlying storage to commit to + } + /** * @param pageId page Id * @return the key to this page @@ -99,6 +111,7 @@ public PageId getKeyFromPageId(PageId pageId) { public void close() { mPageStoreMap.clear(); mPageStoreMap = null; + mPagePool.close(); } /** @@ -130,7 +143,7 @@ public void setPageLength(int pageLength) { } } - private static class PagePool { + private static class PagePool implements Closeable { private final int mPageSize; private final LinkedList mPool = new LinkedList<>(); @@ -154,5 +167,10 @@ public void release(MemPage page) { mPool.push(page); } } + + @Override + public void close() { + mPool.clear(); + } } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java index 4a26dae5ee67..ff987931ba5c 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/MemoryPageStoreDir.java @@ -13,6 +13,7 @@ import static java.util.Objects.requireNonNull; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; import alluxio.client.file.cache.evictor.CacheEvictor; @@ -57,4 +58,9 @@ public void reset() { public void scanPages(Consumer> pageInfoConsumer) { //do nothing } + + @Override + public Optional getUsage() { + return Optional.of(new QuotaManagedPageStoreDir.Usage()); + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java index 0a3378b18588..992b7570a171 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/PageStoreDir.java @@ -12,6 +12,7 @@ package alluxio.client.file.cache.store; import alluxio.client.file.cache.CacheManagerOptions; +import alluxio.client.file.cache.CacheStatus; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; import alluxio.client.file.cache.evictor.CacheEvictor; @@ -33,7 +34,7 @@ /** * Directory of page store. */ -public interface PageStoreDir { +public interface PageStoreDir extends CacheStatus { Logger LOG = LoggerFactory.getLogger(PageStoreDir.class); /** @@ -66,12 +67,6 @@ static PageStoreDir createPageStoreDir(CacheEvictorOptions cacheEvictorOptions, PageStore.create(pageStoreOptions), CacheEvictor.create(cacheEvictorOptions) ); - case ROCKS: - return new RocksPageStoreDir( - pageStoreOptions, - PageStore.create(pageStoreOptions), - CacheEvictor.create(cacheEvictorOptions) - ); case MEM: return new MemoryPageStoreDir( pageStoreOptions, @@ -168,6 +163,11 @@ static void clear(Path rootPath) throws IOException { */ boolean reserve(long bytes); + /** + * @param bytes + */ + void deleteTempPage(PageInfo bytes); + /** * @param bytes * @return the bytes used after release diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java index 4de49a2aff24..640a81a12017 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/QuotaManagedPageStoreDir.java @@ -13,14 +13,20 @@ import static com.google.common.base.Preconditions.checkState; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.evictor.CacheEvictor; import alluxio.resource.LockResource; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.concurrent.GuardedBy; @@ -35,6 +41,8 @@ abstract class QuotaManagedPageStoreDir implements PageStoreDir { @GuardedBy("mTempFileIdSetLock") private final Set mTempFileIdSet = new HashSet<>(); + private final Map> mTempFileToPageInfoListMap = new ConcurrentHashMap<>(); + private final Path mRootPath; private final long mCapacityBytes; private final AtomicLong mBytesUsed = new AtomicLong(0); @@ -73,9 +81,12 @@ public void putPage(PageInfo pageInfo) { @Override public void putTempPage(PageInfo pageInfo) { + mTempFileToPageInfoListMap.computeIfAbsent(pageInfo.getPageId().getFileId(), + tempFileId -> new ArrayList<>()).add(pageInfo); try (LockResource lock = new LockResource(mTempFileIdSetLock.readLock())) { mTempFileIdSet.add(pageInfo.getPageId().getFileId()); } + mBytesUsed.addAndGet(pageInfo.getPageSize()); } @Override @@ -84,6 +95,25 @@ public long deletePage(PageInfo pageInfo) { return mBytesUsed.addAndGet(-pageInfo.getPageSize()); } + @Override + public void deleteTempPage(PageInfo pageInfo) { + String fileId = pageInfo.getPageId().getFileId(); + if (mTempFileToPageInfoListMap.containsKey(fileId)) { + List pageInfoList = + mTempFileToPageInfoListMap.get(fileId); + if (pageInfoList != null && pageInfoList.contains(pageInfo)) { + pageInfoList.remove(pageInfo); + if (pageInfoList.isEmpty()) { + mTempFileToPageInfoListMap.remove(fileId); + try (LockResource lock = new LockResource(mTempFileIdSetLock.readLock())) { + mTempFileIdSet.remove(fileId); + } + } + } + } + mBytesUsed.addAndGet(-pageInfo.getPageSize()); + } + @Override public boolean putTempFile(String fileId) { try (LockResource lock = new LockResource(mTempFileIdSetLock.writeLock())) { @@ -142,10 +172,17 @@ public void commit(String fileId, String newFileId) throws IOException { try (LockResource tempFileIdSetlock = new LockResource(mTempFileIdSetLock.writeLock()); LockResource fileIdSetlock = new LockResource(mFileIdSetLock.writeLock())) { checkState(mTempFileIdSet.contains(fileId), "temp file does not exist " + fileId); - checkState(!mFileIdSet.contains(newFileId), "file already committed " + newFileId); + // We need a new interface for {@PageStore} interface to remove all pages of a cached file, + // and remove the fileId from this mFileIdSet. See {@DoraWorker#invalidateCachedFile}. + // Currently, invalidated file is still in this map. + //checkState(!mFileIdSet.contains(newFileId), "file already committed " + newFileId); getPageStore().commit(fileId, newFileId); mTempFileIdSet.remove(fileId); mFileIdSet.add(newFileId); + + mTempFileToPageInfoListMap.get(fileId) + .forEach(pageInfo -> mEvictor.updateOnPut(pageInfo.getPageId())); + mTempFileToPageInfoListMap.remove(fileId); } } @@ -157,4 +194,39 @@ public void abort(String fileId) throws IOException { mTempFileIdSet.remove(fileId); } } + + /** + * Generic implementation of cache usage stats. + * Subclasses may need to override the individual cache stat to reflect their own logic + * of usage accounting. + */ + class Usage implements CacheUsage { + @Override + public long used() { + return getCachedBytes(); + } + + @Override + public long available() { + // TODO(bowen): take reserved bytes into account + return getCapacityBytes() - getCachedBytes(); + } + + @Override + public long capacity() { + return getCapacityBytes(); + } + + /** + * This generic implementation assumes the directory does not support finer-grained + * stats partitioning. + * + * @param partition how to partition the cache + * @return always empty + */ + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + } } diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java index 4526340d668f..1fb83c5394f6 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStore.java @@ -15,6 +15,7 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import alluxio.proto.client.Cache; import com.google.common.base.Preconditions; @@ -177,7 +178,7 @@ public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOEx } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { Preconditions.checkArgument(pageOffset >= 0, "page offset should be non-negative"); try { diff --git a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java index 1f218ca56b44..f3ee48ea9fa6 100644 --- a/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java +++ b/core/client/fs/src/main/java/alluxio/client/file/cache/store/RocksPageStoreDir.java @@ -13,6 +13,7 @@ import static com.google.common.base.Preconditions.checkState; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; @@ -83,4 +84,9 @@ private Optional parsePageInfo(RocksIterator iter) { } return Optional.empty(); } + + @Override + public Optional getUsage() { + return Optional.of(new QuotaManagedPageStoreDir.Usage()); + } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java b/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java index 1df3898b1054..fbebff6edb9b 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java +++ b/core/client/fs/src/test/java/alluxio/client/file/MockFileInStream.java @@ -70,7 +70,7 @@ public long getPos() throws IOException { @Override public int positionedRead(long position, byte[] buffer, int offset, int length) throws IOException { - throw new UnsupportedOperationException("positionedRead not implemented for mock FileInStream"); + return length; } @Override diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java index 5888ca44556d..2a4fd57d8dfd 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/CacheManagerWithShadowCacheTest.java @@ -17,10 +17,11 @@ import alluxio.Constants; import alluxio.client.file.CacheContext; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.util.io.BufferUtils; import org.junit.Before; @@ -32,6 +33,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.Optional; +import java.util.function.Supplier; /** * Tests for the {@link LocalCacheManager} class. @@ -228,7 +231,7 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer buffer, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer buffer, CacheContext cacheContext) { if (!mCache.containsKey(pageId)) { return 0; @@ -241,6 +244,29 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return bytesToRead; } + @Override + public void commitFile(String fileId) { + throw new UnsupportedOperationException("commitFile method is unsupported. "); + } + + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + return bytesRead; + } + byte[] page = externalDataSupplier.get(); + if (page.length == 0) { + return 0; + } + buffer.writeBytes(page, pageOffset, bytesToRead); + put(pageId, page, cacheContext); + return bytesToRead; + } + @Override public boolean delete(PageId pageId) { if (mCache.containsKey(pageId)) { @@ -261,6 +287,27 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac } @Override - public void close() throws Exception {} + public void deleteFile(String fileId) { + // no-op + } + + @Override + public void deleteTempFile(String fileId) { + // no-op + } + + @Override + public Optional getUsage() { + return Optional.empty(); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + return Optional.empty(); + } + + @Override + public void close() {} } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java b/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java index fde5fafbc3c2..9917f03c82f2 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/HangingPageStore.java @@ -12,9 +12,9 @@ package alluxio.client.file.cache; import alluxio.client.file.cache.store.LocalPageStore; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.exception.PageNotFoundException; +import alluxio.file.ReadTargetBuffer; import java.io.IOException; import java.nio.ByteBuffer; @@ -44,7 +44,7 @@ public void delete(PageId pageId) throws IOException, PageNotFoundException { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, boolean isTemporary) throws IOException, PageNotFoundException { checkStopHanging(); diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java index ab17f2a98a7d..910225b940c8 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheFileInStreamTest.java @@ -11,6 +11,8 @@ package alluxio.client.file.cache; +import static org.junit.Assert.fail; + import alluxio.AlluxioURI; import alluxio.Constants; import alluxio.client.file.CacheContext; @@ -20,7 +22,7 @@ import alluxio.client.file.ListStatusPartialResult; import alluxio.client.file.MockFileInStream; import alluxio.client.file.URIStatus; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.client.file.cache.context.CachePerThreadContext; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; @@ -32,6 +34,7 @@ import alluxio.exception.FileIncompleteException; import alluxio.exception.InvalidPathException; import alluxio.exception.OpenDirectoryException; +import alluxio.file.ReadTargetBuffer; import alluxio.grpc.CancelSyncMetadataPResponse; import alluxio.grpc.CheckAccessPOptions; import alluxio.grpc.CreateDirectoryPOptions; @@ -59,6 +62,7 @@ import alluxio.job.JobRequest; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.security.authorization.AclEntry; import alluxio.util.io.BufferUtils; import alluxio.util.io.PathUtils; @@ -95,6 +99,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -123,6 +128,7 @@ public void before() { MetricsSystem.clearAllMetrics(); sConf.set(PropertyKey.USER_CLIENT_CACHE_PAGE_SIZE, mPageSize); sConf.set(PropertyKey.USER_CLIENT_CACHE_IN_STREAM_BUFFER_SIZE, mBufferSize); + CachePerThreadContext.get().setCacheEnabled(true); } @Test @@ -226,7 +232,8 @@ public void readPartialPageThroughReadByteBufferMethod() throws Exception { stream.seek(offset); Assert.assertEquals(partialReadSize, stream.read(cacheMissBuffer)); Assert.assertArrayEquals( - Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheMissBuffer.array()); + Arrays.copyOfRange(testData, offset, offset + partialReadSize), + cacheMissBuffer.array()); Assert.assertEquals(0, manager.mPagesServed); Assert.assertEquals(1, manager.mPagesCached); @@ -234,8 +241,8 @@ public void readPartialPageThroughReadByteBufferMethod() throws Exception { ByteBuffer cacheHitBuffer = ByteBuffer.wrap(new byte[partialReadSize]); stream.seek(offset); Assert.assertEquals(partialReadSize, stream.read(cacheHitBuffer)); - Assert.assertArrayEquals( - Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheHitBuffer.array()); + Assert.assertArrayEquals(Arrays.copyOfRange(testData, offset, offset + partialReadSize), + cacheHitBuffer.array()); Assert.assertEquals(1, manager.mPagesServed); } @@ -363,6 +370,10 @@ public void positionedReadPartialPage() throws Exception { Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheMiss); Assert.assertEquals(0, manager.mPagesServed); Assert.assertEquals(1, manager.mPagesCached); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); // cache hit byte[] cacheHit = new byte[partialReadSize]; @@ -371,6 +382,10 @@ public void positionedReadPartialPage() throws Exception { Assert.assertArrayEquals( Arrays.copyOfRange(testData, offset, offset + partialReadSize), cacheHit); Assert.assertEquals(1, manager.mPagesServed); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); } @Test @@ -382,7 +397,8 @@ public void positionReadOversizedBuffer() throws Exception { // cache miss byte[] cacheMiss = new byte[fileSize * 2]; - Assert.assertEquals(fileSize - 1, stream.positionedRead(1, cacheMiss, 2, fileSize * 2)); + Assert.assertEquals(fileSize - 1, + stream.positionedRead(1, cacheMiss, 2, fileSize * 2)); Assert.assertArrayEquals( Arrays.copyOfRange(testData, 1, fileSize - 1), Arrays.copyOfRange(cacheMiss, 2, fileSize)); @@ -391,7 +407,8 @@ public void positionReadOversizedBuffer() throws Exception { // cache hit byte[] cacheHit = new byte[fileSize * 2]; - Assert.assertEquals(fileSize - 1, stream.positionedRead(1, cacheHit, 2, fileSize * 2)); + Assert.assertEquals(fileSize - 1, + stream.positionedRead(1, cacheHit, 2, fileSize * 2)); Assert.assertArrayEquals( Arrays.copyOfRange(testData, 1, fileSize - 1), Arrays.copyOfRange(cacheHit, 2, fileSize)); @@ -415,15 +432,21 @@ public void readPagesMetrics() throws Exception { MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()).getCount()); Assert.assertEquals(fileSize, MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).getCount()); + Assert.assertEquals(5, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); // cache hit stream.read(); - Assert.assertEquals(1, - MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).getCount()); Assert.assertEquals(readSize, MetricsSystem.meter( MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()).getCount()); Assert.assertEquals(fileSize, MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL.getName()).getCount()); + Assert.assertEquals(5, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); } @Test @@ -438,7 +461,8 @@ public void externalStoreMultiRead() throws Exception { ByteArrayFileSystem fs = new MultiReadByteArrayFileSystem(files); LocalCacheFileInStream stream = new LocalCacheFileInStream(fs.getStatus(testFilename), - (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf); + (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf, + Optional.empty()); // cache miss byte[] cacheMiss = new byte[fileSize]; @@ -467,7 +491,8 @@ public void externalStoreMultiReadThroughReadByteBufferMethod() throws Exception ByteArrayFileSystem fs = new MultiReadByteArrayFileSystem(files); LocalCacheFileInStream stream = new LocalCacheFileInStream(fs.getStatus(testFilename), - (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf); + (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf, + Optional.empty()); // cache miss ByteBuffer cacheMissBuf = ByteBuffer.wrap(new byte[fileSize]); @@ -515,7 +540,7 @@ public void cacheMetricCacheHitReadTime() throws Exception { LocalCacheFileInStream stream = new LocalCacheFileInStream(fs.getStatus(testFileName), (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, - sConf) { + sConf, Optional.empty()) { @Override protected Stopwatch createUnstartedStopwatch() { return Stopwatch.createUnstarted(timeSource); @@ -523,11 +548,8 @@ protected Stopwatch createUnstartedStopwatch() { }; Assert.assertArrayEquals(testData, ByteStreams.toByteArray(stream)); - long timeReadCache = recordedMetrics.get( - MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName()); long timeReadExternal = recordedMetrics.get( MetricKey.CLIENT_CACHE_PAGE_READ_EXTERNAL_TIME_NS.getMetricName()); - Assert.assertEquals(timeSource.get(StepTicker.Type.CACHE_HIT), timeReadCache); Assert.assertEquals(timeSource.get(StepTicker.Type.CACHE_MISS), timeReadExternal); } @@ -563,6 +585,88 @@ public void testUnbuffer() throws Exception { Assert.assertEquals(1, manager.mPagesServed); } + @Test + public void testPageDataFileCorrupted() throws Exception + { + int pages = 10; + int fileSize = mPageSize * pages; + byte[] testData = BufferUtils.getIncreasingByteArray(fileSize); + ByteArrayCacheManager manager = new ByteArrayCacheManager(); + //by default local cache fallback is not enabled, the read should fail for any error + LocalCacheFileInStream streamWithOutFallback = setupWithSingleFile(testData, manager); + + sConf.set(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED, true); + LocalCacheFileInStream streamWithFallback = setupWithSingleFile(testData, manager); + Assert.assertEquals(100, streamWithFallback.positionedRead(0, new byte[10], 100, 100)); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_POSITION_READ_FALLBACK.getName()).getCount()); + } + + @Test + public void testPositionReadFallBack() throws Exception + { + int pages = 10; + int fileSize = mPageSize * pages; + byte[] testData = BufferUtils.getIncreasingByteArray(fileSize); + ByteArrayCacheManager manager = new ByteArrayCacheManager(); + sConf.set(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED, false); + //by default local cache fallback is not enabled, the read should fail for any error + LocalCacheFileInStream streamWithOutFallback = setupWithSingleFile(testData, manager); + try { + streamWithOutFallback.positionedRead(0, new byte[10], 100, 100); + fail("Expect position read fail here."); + } catch (ArrayIndexOutOfBoundsException e) { + //expected exception + } + sConf.set(PropertyKey.USER_CLIENT_CACHE_FALLBACK_ENABLED, true); + LocalCacheFileInStream streamWithFallback = setupWithSingleFile(testData, manager); + Assert.assertEquals(100, streamWithFallback.positionedRead(0, new byte[10], 100, 100)); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_POSITION_READ_FALLBACK.getName()).getCount()); + } + + @Test + public void testPositionReadWithPerThreadContextSameThread() throws Exception + { + int fileSize = mPageSize * 5; + byte[] testData = BufferUtils.getIncreasingByteArray(fileSize); + ByteArrayCacheManager manager = new ByteArrayCacheManager(); + LocalCacheFileInStream stream = setupWithSingleFile(testData, manager); + + // read last page with cache enabled (cache miss) + CachePerThreadContext.get().setCacheEnabled(true); + stream.positionedRead(fileSize - mPageSize, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(0, + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + // read last page with cache enabled (cache hit) + stream.positionedRead(fileSize - mPageSize, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + + MetricsSystem.clearAllMetrics(); + // read first page with cache disabled (cache miss) + CachePerThreadContext.get().setCacheEnabled(false); + stream.positionedRead(0, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(0, + MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).getCount()); + Assert.assertEquals(1, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + // read last page again with cache enabled (cache miss) + stream.positionedRead(0, new byte[mPageSize], 0, mPageSize); + Assert.assertEquals(2, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_EXTERNAL_REQUESTS.getName()).getCount()); + Assert.assertEquals(0, + MetricsSystem.counter(MetricKey.CLIENT_CACHE_HIT_REQUESTS.getName()).getCount()); + } + private LocalCacheFileInStream setupWithSingleFile(byte[] data, CacheManager manager) throws Exception { Map files = new HashMap<>(); @@ -572,7 +676,8 @@ private LocalCacheFileInStream setupWithSingleFile(byte[] data, CacheManager man ByteArrayFileSystem fs = new ByteArrayFileSystem(files); return new LocalCacheFileInStream(fs.getStatus(testFilename), - (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf); + (status) -> fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, sConf, + Optional.empty()); } private Map setupWithMultipleFiles(Map files, @@ -587,7 +692,7 @@ private Map setupWithMultipleFiles(Map fs.openFile(status, OpenFilePOptions.getDefaultInstance()), manager, - sConf)); + sConf, Optional.empty())); } catch (Exception e) { // skip } @@ -599,6 +704,7 @@ private URIStatus generateURIStatus(String path, long len) { FileInfo info = new FileInfo(); info.setFileId(path.hashCode()); info.setPath(path); + info.setUfsPath(path); info.setLength(len); return new URIStatus(info); } @@ -668,7 +774,7 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { if (!mPages.containsKey(pageId)) { return 0; @@ -678,6 +784,29 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return bytesToRead; } + @Override + public void commitFile(String fileId) { + throw new UnsupportedOperationException("commitFile method is unsupported. "); + } + + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + return bytesRead; + } + byte[] page = externalDataSupplier.get(); + if (page.length == 0) { + return 0; + } + buffer.writeBytes(page, pageOffset, bytesToRead); + put(pageId, page, cacheContext); + return bytesToRead; + } + @Override public boolean delete(PageId pageId) { return mPages.remove(pageId) != null; @@ -697,6 +826,49 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac public void close() throws Exception { // no-op } + + @Override + public void deleteFile(String fileId) { + // no-op + } + + @Override + public void deleteTempFile(String fileId) { + // no-op + } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + return Optional.empty(); + } + + class Usage implements CacheUsage { + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + + @Override + public long used() { + return mPages.values().stream().mapToInt(page -> page.length).sum(); + } + + @Override + public long available() { + return Integer.MAX_VALUE; + } + + @Override + public long capacity() { + return Integer.MAX_VALUE; + } + } } /** @@ -1036,7 +1208,7 @@ public TimedMockByteArrayCacheManager(StepTicker ticker) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { int read = super.get(pageId, pageOffset, bytesToRead, target, cacheContext); if (read > 0) { diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java index f564d4035ed2..0e3798f5d9cc 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/LocalCacheManagerTest.java @@ -36,8 +36,13 @@ import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; import alluxio.exception.status.ResourceExhaustedException; +import alluxio.file.ByteArrayTargetBuffer; +import alluxio.file.NettyBufTargetBuffer; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import alluxio.util.CommonUtils; import alluxio.util.WaitForOptions; import alluxio.util.io.BufferUtils; @@ -46,6 +51,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.DefaultFileRegion; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -669,6 +677,10 @@ public void asyncRestoreReadOnly() throws Exception { assertTrue(mCacheManager.delete(PAGE_ID2)); } + /** + * Invalid page file will be deleted and cache manager will start normally. + * @throws Exception + */ @Test public void syncRestoreUnknownFile() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_ASYNC_RESTORE_ENABLED, false); @@ -685,10 +697,14 @@ public void syncRestoreUnknownFile() throws Exception { mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); mCacheManager = LocalCacheManager.create(mCacheManagerOptions, mPageMetaStore); assertEquals(CacheManager.State.READ_WRITE, mCacheManager.state()); - assertEquals(0, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); - assertEquals(0, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); + assertEquals(PAGE1.length, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); + assertEquals(PAGE2.length, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); } + /** + * Invalid page file will be deleted and cache manager will start normally. + * @throws Exception + */ @Test public void asyncRestoreUnknownFile() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_ASYNC_RESTORE_ENABLED, true); @@ -703,8 +719,8 @@ public void asyncRestoreUnknownFile() throws Exception { FileUtils.createFile(Paths.get(rootDir, "invalidPageFile").toString()); mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); mCacheManager = createLocalCacheManager(mConf, mPageMetaStore); - assertEquals(0, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); - assertEquals(0, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); + assertEquals(PAGE1.length, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0)); + assertEquals(PAGE2.length, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); } @Test @@ -806,6 +822,29 @@ public void asyncRestoreWithMorePagesThanCapacity() throws Exception { assertEquals(0, mCacheManager.get(pageUuid, PAGE2.length, mBuf, 0)); } + @Test + public void ttlDeleteOldPagesWhenRestore() throws Exception { + mConf.set(PropertyKey.USER_CLIENT_CACHE_TTL_THRESHOLD_SECONDS, 5); + mConf.set(PropertyKey.USER_CLIENT_CACHE_TTL_ENABLED, true); + mCacheManagerOptions = CacheManagerOptions.create(mConf); + mCacheManager.close(); + PageStoreDir dir = PageStoreDir.createPageStoreDirs(mCacheManagerOptions) + .get(0); // previous page store has been closed + PageId pageUuid = new PageId(UUID.randomUUID().toString(), 0); + dir.getPageStore().put(PAGE_ID1, PAGE1); + dir.getPageStore().put(PAGE_ID2, PAGE2); + + dir = PageStoreDir.createPageStoreDirs(mCacheManagerOptions).get(0); + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + Thread.sleep(6000); + dir.getPageStore().put(pageUuid, + BufferUtils.getIncreasingByteArray(PAGE1.length + PAGE2.length + 1)); + mCacheManager = createLocalCacheManager(mConf, mPageMetaStore); + assertFalse(mCacheManager.hasPageUnsafe(PAGE_ID1)); + assertFalse(mCacheManager.hasPageUnsafe(PAGE_ID2)); + assertTrue(mCacheManager.hasPageUnsafe(pageUuid)); // we should have the new page + } + @Test public void asyncCache() throws Exception { // this must be smaller than the number of locks in the page store for the test to succeed @@ -965,6 +1004,58 @@ public void getTimeout() throws Exception { pageStore.setGetHanging(false); } + @Test + public void getFaultyReadWithNoExceptionManager() throws Exception { + PageStoreOptions pageStoreOptions = PageStoreOptions.create(mConf).get(0); + FaultyPageStore pageStore = new FaultyPageStore(); + PageStoreDir dir = + new LocalPageStoreDir(pageStoreOptions, pageStore, mEvictor); + + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + NoExceptionCacheManager cacheManager = + new NoExceptionCacheManager(createLocalCacheManager(mConf, mPageMetaStore)); + cacheManager.put(PAGE_ID1, PAGE1); + ByteArrayTargetBuffer targetBuffer = new ByteArrayTargetBuffer(mBuf, 0); + pageStore.setGetFaulty(true); + assertEquals(-1, cacheManager.get(PAGE_ID1, PAGE1.length, + targetBuffer, CacheContext.defaults())); + assertEquals(0, targetBuffer.offset()); + } + + @Test + public void getFaultyReadWithLocalCacheManager() throws Exception { + PageStoreOptions pageStoreOptions = PageStoreOptions.create(mConf).get(0); + FaultyPageStore pageStore = new FaultyPageStore(); + PageStoreDir dir = + new LocalPageStoreDir(pageStoreOptions, pageStore, mEvictor); + + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + LocalCacheManager cacheManager = createLocalCacheManager(mConf, mPageMetaStore); + cacheManager.put(PAGE_ID1, PAGE1); + ByteArrayTargetBuffer targetBuffer = new ByteArrayTargetBuffer(mBuf, 0); + pageStore.setGetFaulty(true); + assertEquals(-1, cacheManager.get(PAGE_ID1, PAGE1.length, + targetBuffer, CacheContext.defaults())); + assertEquals(0, targetBuffer.offset()); + } + + @Test + public void getCorruptedReadWithLocalCacheManager() throws Exception { + PageStoreOptions pageStoreOptions = PageStoreOptions.create(mConf).get(0); + FaultyPageStore pageStore = new FaultyPageStore(); + PageStoreDir dir = + new LocalPageStoreDir(pageStoreOptions, pageStore, mEvictor); + + mPageMetaStore = new DefaultPageMetaStore(ImmutableList.of(dir)); + LocalCacheManager cacheManager = createLocalCacheManager(mConf, mPageMetaStore); + cacheManager.put(PAGE_ID1, PAGE1); + ByteArrayTargetBuffer targetBuffer = new ByteArrayTargetBuffer(mBuf, 0); + pageStore.setGetCorrupted(true); + assertEquals(-1, cacheManager.get(PAGE_ID1, PAGE1.length, + targetBuffer, CacheContext.defaults())); + assertEquals(0, targetBuffer.offset()); + } + @Test public void deleteTimeout() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_TIMEOUT_DURATION, "2s"); @@ -1090,6 +1181,28 @@ public void listPageIds() throws Exception { 0).get(0)); } + @Test + public void getDataFileChannel() throws Exception { + mCacheManager = createLocalCacheManager(); + mCacheManager.put(PAGE_ID1, PAGE1); + CacheContext cacheContext = CacheContext.defaults(); + Optional dataFileChannel = mCacheManager.getDataFileChannel(PAGE_ID1, + 0, PAGE1.length, cacheContext); + assertNotNull(dataFileChannel); + assertEquals(dataFileChannel.isPresent(), true); + assertEquals(dataFileChannel.get().getNettyOutput() instanceof DefaultFileRegion, true); + DefaultFileRegion defaultFileRegion = + (DefaultFileRegion) dataFileChannel.get().getNettyOutput(); + ByteBuf buf = Unpooled.buffer(PAGE1.length); + NettyBufTargetBuffer targetBuffer = new NettyBufTargetBuffer(buf); + long bytesTransferred = defaultFileRegion.transferTo(targetBuffer.byteChannel(), 0); + assertEquals(bytesTransferred, PAGE1.length); + + byte[] bytes = new byte[PAGE1.length]; + buf.readBytes(bytes); + assertArrayEquals(PAGE1, bytes); + } + /** * A PageStore where put can throw IOException on put or delete. */ @@ -1100,6 +1213,23 @@ public FaultyPageStore() { private AtomicBoolean mPutFaulty = new AtomicBoolean(false); private AtomicBoolean mDeleteFaulty = new AtomicBoolean(false); + private AtomicBoolean mGetFaulty = new AtomicBoolean(false); + + private AtomicBoolean mGetCorrupted = new AtomicBoolean(false); + + @Override + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, + boolean isTemporary) throws IOException, PageNotFoundException { + if (mGetFaulty.get()) { + target.offset(target.offset() + 100); + throw new IOException("Page read fault"); + } + if (mGetCorrupted.get()) { + target.offset(target.offset() + 100); + throw new PageCorruptedException("page corrupted"); + } + return super.get(pageId, pageOffset, bytesToRead, target, isTemporary); + } @Override public void put(PageId pageId, ByteBuffer page, boolean isTemporary) throws IOException { @@ -1124,6 +1254,14 @@ void setPutFaulty(boolean faulty) { void setDeleteFaulty(boolean faulty) { mDeleteFaulty.set(faulty); } + + void setGetFaulty(boolean faulty) { + mGetFaulty.set(faulty); + } + + void setGetCorrupted(boolean faulty) { + mGetCorrupted.set(faulty); + } } /** diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java index 76cbae4ab79e..422517afad50 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/TimeBoundPageStoreTest.java @@ -20,12 +20,12 @@ import static org.junit.Assert.fail; import alluxio.Constants; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; import alluxio.exception.PageNotFoundException; +import alluxio.file.ByteArrayTargetBuffer; import alluxio.util.io.BufferUtils; import org.junit.Before; diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java index 3c47646f64b9..3adaeda7f3ad 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/LocalPageStoreTest.java @@ -15,10 +15,13 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; +import alluxio.file.ByteArrayTargetBuffer; import org.junit.Before; import org.junit.Rule; @@ -170,12 +173,26 @@ public void cleanFileAndDirectory() throws Exception { assertFalse(Files.exists(p.getParent())); } + @Test + public void testCorruptedPages() throws Exception { + mOptions.setFileBuckets(1); + LocalPageStore pageStore = new LocalPageStore(mOptions); + byte[] buf = new byte[1000]; + PageId id = new PageId("1", 0); + pageStore.put(id, "corrupted".getBytes()); + assertThrows(PageCorruptedException.class, () -> { + //the bytes caller want to read is larger than the page file, mostly means the page corrupted + pageStore.get(id, 0, 100, new ByteArrayTargetBuffer(buf, 0)); + }); + } + private void helloWorldTest(PageStore store) throws Exception { String msg = "Hello, World!"; PageId id = new PageId("0", 0); store.put(id, msg.getBytes()); byte[] buf = new byte[1024]; - assertEquals(msg.getBytes().length, store.get(id, new ByteArrayTargetBuffer(buf, 0))); + assertEquals(msg.getBytes().length, store.get(id, 0, msg.length(), + new ByteArrayTargetBuffer(buf, 0))); assertArrayEquals(msg.getBytes(), Arrays.copyOfRange(buf, 0, msg.getBytes().length)); } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java index 44cae0a0234e..e20f11f50e5f 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/MemoryPageStoreTest.java @@ -16,6 +16,7 @@ import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.file.ByteArrayTargetBuffer; import org.junit.Before; import org.junit.Test; @@ -42,7 +43,8 @@ private void helloWorldTest(PageStore store) throws Exception { PageId id = new PageId("0", 0); store.put(id, msg.getBytes()); byte[] buf = new byte[PAGE_SIZE]; - assertEquals(msg.getBytes().length, store.get(id, new ByteArrayTargetBuffer(buf, 0))); + assertEquals(msg.getBytes().length, + store.get(id, 0, msg.length(), new ByteArrayTargetBuffer(buf, 0))); assertArrayEquals(msg.getBytes(), Arrays.copyOfRange(buf, 0, msg.getBytes().length)); } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java index 3c6e768f5903..17e16c486c3b 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreDirTest.java @@ -11,11 +11,14 @@ package alluxio.client.file.cache.store; +import static alluxio.client.file.cache.CacheUsage.PartitionDescriptor.file; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import alluxio.ProjectConstants; import alluxio.client.file.cache.CacheManagerOptions; +import alluxio.client.file.cache.CacheUsage; +import alluxio.client.file.cache.CacheUsageView; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.conf.AlluxioConfiguration; @@ -34,17 +37,19 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.UUID; @RunWith(Parameterized.class) public class PageStoreDirTest { + public static final long CACHE_CAPACITY = 65536; + public static final long PAGE_SIZE = 1024; private final AlluxioConfiguration mConf = Configuration.global(); - @Parameterized.Parameters + @Parameterized.Parameters(name = "{index}-{0}") public static Collection data() { return Arrays.asList(new Object[][] { - {PageStoreType.ROCKS}, {PageStoreType.LOCAL}, {PageStoreType.MEM} }); @@ -65,8 +70,8 @@ public void before() throws Exception { CacheManagerOptions cacheManagerOptions = CacheManagerOptions.create(mConf); mOptions = cacheManagerOptions.getPageStoreOptions().get(0); mOptions.setStoreType(mPageStoreType); - mOptions.setPageSize(1024); - mOptions.setCacheSize(65536); + mOptions.setPageSize(PAGE_SIZE); + mOptions.setCacheSize(CACHE_CAPACITY); mOptions.setAlluxioVersion(ProjectConstants.VERSION); mOptions.setRootDir(Paths.get(mTemp.getRoot().getAbsolutePath())); @@ -120,4 +125,26 @@ public void getPagesUUID() throws Exception { assertEquals(pages, restored); } } + + @Test + public void cacheUsage() throws Exception { + int len = 32; + int count = 16; + byte[] data = BufferUtils.getIncreasingByteArray(len); + for (int i = 0; i < count; i++) { + PageId id = new PageId("0", i); + mPageStoreDir.getPageStore().put(id, data); + mPageStoreDir.putPage(new PageInfo(id, data.length, mPageStoreDir)); + } + Optional usage = mPageStoreDir.getUsage(); + assertEquals(Optional.of(mPageStoreDir.getCapacityBytes()), + usage.map(CacheUsageView::capacity)); + assertEquals(Optional.of((long) len * count), usage.map(CacheUsageView::used)); + assertEquals(Optional.of(mPageStoreDir.getCapacityBytes() - (long) len * count), + usage.map(CacheUsageView::available)); + // cache dir currently does not support get file level usage stat + Optional fileUsage = mPageStoreDir.getUsage() + .flatMap(usage1 -> usage1.partitionedBy(file("0"))); + assertEquals(Optional.empty(), fileUsage); + } } diff --git a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java index e4b5cc115159..3c382d5b4f22 100644 --- a/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java +++ b/core/client/fs/src/test/java/alluxio/client/file/cache/store/PageStoreTest.java @@ -19,7 +19,9 @@ import alluxio.ProjectConstants; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageStore; +import alluxio.exception.PageCorruptedException; import alluxio.exception.PageNotFoundException; +import alluxio.file.ByteArrayTargetBuffer; import alluxio.util.io.BufferUtils; import org.junit.After; @@ -40,7 +42,6 @@ public class PageStoreTest { @Parameterized.Parameters public static Collection data() { return Arrays.asList(new Object[][] { - {PageStoreType.ROCKS}, {PageStoreType.LOCAL}, {PageStoreType.MEM} }); @@ -79,7 +80,8 @@ public void helloWorldTest() throws Exception { PageId id = new PageId("0", 0); mPageStore.put(id, msgBytes); byte[] buf = new byte[1024]; - assertEquals(msgBytes.length, mPageStore.get(id, new ByteArrayTargetBuffer(buf, 0))); + assertEquals(msgBytes.length, + mPageStore.get(id, 0, msgBytes.length, new ByteArrayTargetBuffer(buf, 0))); assertArrayEquals(msgBytes, Arrays.copyOfRange(buf, 0, msgBytes.length)); mPageStore.delete(id); try { @@ -97,7 +99,8 @@ public void getOffset() throws Exception { mPageStore.put(id, BufferUtils.getIncreasingByteArray(len)); byte[] buf = new byte[len]; for (int offset = 1; offset < len; offset++) { - int bytesRead = mPageStore.get(id, offset, len, new ByteArrayTargetBuffer(buf, 0), false); + int bytesRead = mPageStore.get(id, offset, len - offset, + new ByteArrayTargetBuffer(buf, 0), false); assertEquals(len - offset, bytesRead); assertArrayEquals(BufferUtils.getIncreasingByteArray(offset, len - offset), Arrays.copyOfRange(buf, 0, bytesRead)); @@ -111,7 +114,7 @@ public void getOffsetOverflow() throws Exception { PageId id = new PageId("0", 0); mPageStore.put(id, BufferUtils.getIncreasingByteArray(len)); byte[] buf = new byte[1024]; - assertThrows(IllegalArgumentException.class, () -> + assertThrows(PageCorruptedException.class, () -> mPageStore.get(id, offset, len, new ByteArrayTargetBuffer(buf, 0))); } diff --git a/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java b/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java index 5b0c74fd6fe7..372bc3604033 100644 --- a/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java +++ b/core/client/hdfs/src/main/java/alluxio/hadoop/LocalCacheFileSystem.java @@ -16,12 +16,14 @@ import alluxio.AlluxioURI; import alluxio.client.file.CacheContext; +import alluxio.client.file.FileInStream; import alluxio.client.file.URIStatus; import alluxio.client.file.cache.CacheManager; import alluxio.client.file.cache.LocalCacheFileInStream; import alluxio.client.file.cache.filter.CacheFilter; import alluxio.conf.AlluxioConfiguration; import alluxio.conf.PropertyKey; +import alluxio.exception.AlluxioException; import alluxio.metrics.MetricsConfig; import alluxio.metrics.MetricsSystem; import alluxio.wire.FileInfo; @@ -41,6 +43,7 @@ import java.io.IOException; import java.net.URI; import java.util.Map; +import java.util.Optional; import java.util.Properties; /** @@ -93,8 +96,8 @@ public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuratio } MetricsSystem.startSinksFromConfig(new MetricsConfig(metricsProperties)); mCacheManager = CacheManager.Factory.get(mAlluxioConf); - LocalCacheFileInStream.registerMetrics(); mCacheFilter = CacheFilter.create(mAlluxioConf); + LocalCacheFileInStream.registerMetrics(); } @Override @@ -146,19 +149,45 @@ public FSDataInputStream open(Path path, int bufferSize) throws IOException { } /** - * Attempts to open the specified file for reading. + * A wrapper method to default not enforce an open call. * * @param status the status of the file to open * @param bufferSize stream buffer size in bytes, currently unused * @return an {@link FSDataInputStream} at the indicated path of a file */ public FSDataInputStream open(URIStatus status, int bufferSize) throws IOException { + return open(status, bufferSize, false); + } + + /** + * Attempts to open the specified file for reading. + * + * @param status the status of the file to open + * @param bufferSize stream buffer size in bytes, currently unused + * @param enforceOpen flag to enforce calling open to external storage + * @return an {@link FSDataInputStream} at the indicated path of a file + */ + public FSDataInputStream open(URIStatus status, int bufferSize, boolean enforceOpen) + throws IOException { if (mCacheManager == null || !mCacheFilter.needsCache(status)) { return mExternalFileSystem.open(HadoopUtils.toPath(new AlluxioURI(status.getPath())), bufferSize); } + Optional externalFileInStream; + if (enforceOpen) { + try { + // making the open call right now, instead of later when called back + externalFileInStream = Optional.of(mAlluxioFileOpener.open(status)); + } catch (AlluxioException e) { + throw new IOException(e); + } + } else { + externalFileInStream = Optional.empty(); + } + return new FSDataInputStream(new HdfsFileInputStream( - new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf), + new LocalCacheFileInStream(status, mAlluxioFileOpener, mCacheManager, mAlluxioConf, + externalFileInStream), statistics)); } diff --git a/core/common/pom.xml b/core/common/pom.xml index 7834960d54d4..c4082e137d33 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -95,10 +95,26 @@ io.grpc grpc-stub + + io.prometheus + prometheus-metrics-core + + + io.prometheus + prometheus-metrics-exporter-servlet-jakarta + + + io.prometheus + prometheus-metrics-instrumentation-jvm + io.swagger swagger-annotations + + jakarta.servlet + jakarta.servlet-api + org.apache.commons commons-lang3 @@ -119,6 +135,10 @@ org.apache.zookeeper zookeeper + + org.eclipse.jetty + jetty-servlet + org.reflections reflections diff --git a/core/common/src/main/java/alluxio/conf/PropertyKey.java b/core/common/src/main/java/alluxio/conf/PropertyKey.java index dbf013f7b959..ca03396c0300 100755 --- a/core/common/src/main/java/alluxio/conf/PropertyKey.java +++ b/core/common/src/main/java/alluxio/conf/PropertyKey.java @@ -6042,6 +6042,13 @@ public String toString() { .setConsistencyCheckLevel(ConsistencyCheckLevel.WARN) .setScope(Scope.CLIENT) .build(); + public static final PropertyKey USER_CLIENT_CACHE_FALLBACK_ENABLED = + booleanBuilder(Name.USER_CLIENT_CACHE_FALLBACK_ENABLED) + .setDefaultValue(false) + .setDescription("If this is enabled, local cache will fallback to external storage.") + .setConsistencyCheckLevel(ConsistencyCheckLevel.WARN) + .setScope(Scope.CLIENT) + .build(); public static final PropertyKey USER_CLIENT_CACHE_FILTER_CLASS = classBuilder(Name.USER_CLIENT_CACHE_FILTER_CLASS) .setDefaultValue("alluxio.client.file.cache.filter.DefaultCacheFilter") @@ -8889,6 +8896,8 @@ public static final class Name { "alluxio.user.client.cache.async.write.threads"; public static final String USER_CLIENT_CACHE_ENABLED = "alluxio.user.client.cache.enabled"; + public static final String USER_CLIENT_CACHE_FALLBACK_ENABLED = + "alluxio.user.client.cache.fallback.enabled"; public static final String USER_CLIENT_CACHE_FILTER_CLASS = "alluxio.user.client.cache.filter.class"; public static final String USER_CLIENT_CACHE_FILTER_CONFIG_FILE = diff --git a/core/common/src/main/java/alluxio/exception/PageCorruptedException.java b/core/common/src/main/java/alluxio/exception/PageCorruptedException.java new file mode 100644 index 000000000000..45f4be35cac9 --- /dev/null +++ b/core/common/src/main/java/alluxio/exception/PageCorruptedException.java @@ -0,0 +1,35 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.exception; + +/** + * An exception that should be thrown when the data of a page has been corrupted in store. + */ +public class PageCorruptedException extends RuntimeException { + + /** + * Construct PageCorruptedException with the specified message. + * @param message + */ + public PageCorruptedException(String message) { + super(message); + } + + /** + * Construct PageCorruptedException with the specified message and cause. + * @param message + * @param cause + */ + public PageCorruptedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java b/core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java new file mode 100644 index 000000000000..4dd4d5360e62 --- /dev/null +++ b/core/common/src/main/java/alluxio/file/ByteArrayTargetBuffer.java @@ -0,0 +1,105 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import alluxio.annotation.SuppressFBWarnings; +import alluxio.util.io.ChannelAdapters; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +/** + * Target buffer backed by bytes array for zero-copy read from page store. + */ +@SuppressFBWarnings( + value = "EI_EXPOSE_REP2", + justification = "The target byte array is exposed as we expect.") +public class ByteArrayTargetBuffer implements ReadTargetBuffer { + private final byte[] mTarget; + private int mOffset; + + /** + * Constructor. + * @param target + * @param offset + */ + public ByteArrayTargetBuffer(byte[] target, int offset) { + mTarget = target; + mOffset = offset; + } + + @Override + public byte[] byteArray() { + return mTarget; + } + + @Override + public ByteBuffer byteBuffer() { + return ByteBuffer.wrap(mTarget); + } + + @Override + public int offset() { + return mOffset; + } + + @Override + public void offset(int newOffset) { + mOffset = newOffset; + } + + @Override + public long remaining() { + return mTarget.length - mOffset; + } + + @Override + public void writeBytes(byte[] srcArray, int srcOffset, int length) { + System.arraycopy(srcArray, srcOffset, mTarget, mOffset, length); + mOffset += length; + } + + @Override + public void writeBytes(ByteBuf buf) { + int bytesToRead = Math.min(buf.readableBytes(), mTarget.length - mOffset); + buf.readBytes(mTarget, mOffset, bytesToRead); + mOffset += bytesToRead; + } + + @Override + public int readFromFile(RandomAccessFile file, int length) throws IOException { + int bytesRead = file.read(mTarget, mOffset, length); + if (bytesRead != -1) { + mOffset += bytesRead; + } + return bytesRead; + } + + @Override + public int readFromInputStream(InputStream is, int length) throws IOException { + int bytesRead = is.read(mTarget, mOffset, length); + if (bytesRead != -1) { + mOffset += bytesRead; + } + return bytesRead; + } + + @Override + public WritableByteChannel byteChannel() { + return ChannelAdapters.intoByteArray(mTarget, mOffset, mTarget.length - mOffset); + } +} diff --git a/core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java b/core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java new file mode 100644 index 000000000000..20734be67cdf --- /dev/null +++ b/core/common/src/main/java/alluxio/file/ByteBufferTargetBuffer.java @@ -0,0 +1,114 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import alluxio.util.io.ChannelAdapters; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Target buffer backed by nio ByteBuffer for zero-copy read from page store. + */ +public class ByteBufferTargetBuffer implements ReadTargetBuffer { + private final ByteBuffer mTarget; + + /** + * Constructor. + * @param target + */ + public ByteBufferTargetBuffer(ByteBuffer target) { + mTarget = target; + } + + @Override + public byte[] byteArray() { + if (mTarget.hasArray()) { + return mTarget.array(); + } + throw new UnsupportedOperationException("ByteBuffer is not backed by an array"); + } + + @Override + public ByteBuffer byteBuffer() { + return mTarget; + } + + @Override + public int offset() { + return mTarget.position(); + } + + @Override + public void offset(int newOffset) { + mTarget.position(newOffset); + } + + @Override + public WritableByteChannel byteChannel() { + return ChannelAdapters.intoByteBuffer(mTarget); + } + + @Override + public long remaining() { + return mTarget.remaining(); + } + + @Override + public void writeBytes(byte[] srcArray, int srcOffset, int length) { + mTarget.put(srcArray, srcOffset, length); + } + + @Override + public void writeBytes(ByteBuf buf) { + if (mTarget.remaining() <= buf.readableBytes()) { + buf.readBytes(mTarget); + return; + } + int oldLimit = mTarget.limit(); + mTarget.limit(mTarget.position() + buf.readableBytes()); + buf.readBytes(mTarget); + mTarget.limit(oldLimit); + } + + @Override + public int readFromFile(RandomAccessFile file, int length) throws IOException { + int bytesToRead = Math.min(length, mTarget.remaining()); + ByteBuffer slice = mTarget.slice(); + slice.limit(bytesToRead); + int bytesRead = file.getChannel().read(slice); + if (bytesRead > 0) { + mTarget.position(mTarget.position() + bytesRead); + } + return bytesRead; + } + + @Override + public int readFromInputStream(InputStream is, int length) throws IOException { + int bytesToRead = Math.min(length, mTarget.remaining()); + ReadableByteChannel source = Channels.newChannel(is); + ByteBuffer slice = mTarget.slice(); + slice.limit(bytesToRead); + int bytesRead = source.read(slice); + if (bytesRead > 0) { + mTarget.position(mTarget.position() + bytesRead); + } + return bytesRead; + } +} diff --git a/core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java b/core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java new file mode 100644 index 000000000000..0dc2b5f219d6 --- /dev/null +++ b/core/common/src/main/java/alluxio/file/NettyBufTargetBuffer.java @@ -0,0 +1,120 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Netty Buf backed target buffer for zero-copy read from page store. + */ +public class NettyBufTargetBuffer implements ReadTargetBuffer { + private final ByteBuf mTarget; + + /** + * @param target target buffer + */ + public NettyBufTargetBuffer(ByteBuf target) { + mTarget = target; + } + + @Override + public byte[] byteArray() { + return mTarget.array(); + } + + @Override + public ByteBuffer byteBuffer() { + throw new UnsupportedOperationException(); + } + + @Override + public int offset() { + return mTarget.writerIndex(); + } + + @Override + public void offset(int newOffset) { + mTarget.writerIndex(newOffset); + } + + @Override + public WritableByteChannel byteChannel() { + return new WritableByteChannel() { + @Override + public int write(ByteBuffer src) throws IOException { + int readableBytes = src.remaining(); + mTarget.writeBytes(src); + return readableBytes - src.remaining(); + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() throws IOException { + } + }; + } + + @Override + public long remaining() { + return mTarget.writableBytes(); + } + + @Override + public void writeBytes(byte[] srcArray, int srcOffset, int length) { + mTarget.writeBytes(srcArray, srcOffset, length); + } + + @Override + public void writeBytes(ByteBuf buf) { + mTarget.writeBytes(buf); + } + + @Override + public int readFromFile(RandomAccessFile file, int length) throws IOException { + try (FileChannel channel = file.getChannel()) { + return mTarget.writeBytes(channel, length); + } + } + + @Override + public int readFromInputStream(InputStream is, int length) throws IOException { + int bytesToRead = Math.min(length, mTarget.writableBytes()); + ReadableByteChannel source = Channels.newChannel(is); + ByteBuffer slice = mTarget.nioBuffer(mTarget.writerIndex(), bytesToRead); + int bytesRead = source.read(slice); + if (bytesRead > 0) { + mTarget.writerIndex(mTarget.writerIndex() + bytesRead); + } + return bytesRead; + } + + /** + * Get the internal Bytebuf object. + * @return the internal Bytebuf object + */ + public ByteBuf getTargetBuffer() { + return mTarget; + } +} diff --git a/core/common/src/main/java/alluxio/file/ReadTargetBuffer.java b/core/common/src/main/java/alluxio/file/ReadTargetBuffer.java new file mode 100644 index 000000000000..97ec000baaa8 --- /dev/null +++ b/core/common/src/main/java/alluxio/file/ReadTargetBuffer.java @@ -0,0 +1,86 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.file; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +/** + * TargetBuffer for zero-copy read from page store. + */ +public interface ReadTargetBuffer { + + /** + * @return the byte array + */ + byte[] byteArray(); + + /** + * + * @return the byte buffer + */ + ByteBuffer byteBuffer(); + + /** + * @return offset in the buffer + */ + int offset(); + + /** + * Sets the new offset in the buffer. + * + * @param newOffset the new offset + */ + void offset(int newOffset); + + /** + * @return the writable channel + */ + WritableByteChannel byteChannel(); + + /** + * @return the remaining for this buffer + */ + long remaining(); + + /** + * @param srcArray + * @param srcOffset + * @param length + */ + void writeBytes(byte[] srcArray, int srcOffset, int length); + + /** + * @param buf + */ + void writeBytes(ByteBuf buf); + + /** + * @param file + * @param length + * @return bytes read from the file + */ + int readFromFile(RandomAccessFile file, int length) throws IOException; + + /** + * @param is + * @param length + * @return bytes read from input stream + * @throws IOException + */ + int readFromInputStream(InputStream is, int length) throws IOException; +} diff --git a/core/common/src/main/java/alluxio/metrics/MetricKey.java b/core/common/src/main/java/alluxio/metrics/MetricKey.java index 0f9100f9d2d1..5dc399797b80 100644 --- a/core/common/src/main/java/alluxio/metrics/MetricKey.java +++ b/core/common/src/main/java/alluxio/metrics/MetricKey.java @@ -2398,6 +2398,12 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.METER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_EXTERNAL_REQUESTS = + new Builder("Client.CacheExternalRequests") + .setDescription("Total number of requests to read from external storage.") + .setMetricType(MetricType.COUNTER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS = new Builder("Client.CachePageReadCacheTimeNanos") .setDescription("Time in nanoseconds taken to read a page from the client cache " @@ -2412,12 +2418,24 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.METER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_POSITION_READ_FALLBACK = + new Builder("Client.CacheBytesPositionReadFallback") + .setDescription("Total number of position read fallback to external storage.") + .setMetricType(MetricType.COUNTER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_BYTES_DISCARDED = new Builder("Client.CacheBytesDiscarded") .setDescription("Total number of bytes discarded when restoring the page store.") .setMetricType(MetricType.METER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_PAGES_INVALIDATED = + new Builder("Client.CachePagesInvalidated") + .setDescription("Total number of pages invalidated by TTL rules") + .setMetricType(MetricType.METER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_BYTES_EVICTED = new Builder("Client.CacheBytesEvicted") .setDescription("Total number of bytes evicted from the client cache.") @@ -2430,6 +2448,12 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.COUNTER) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_PAGES_AGES = + new Builder("Client.CachePagesAges") + .setDescription("The ages of pages in the client cache.") + .setMetricType(MetricType.HISTOGRAM) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_PAGES_DISCARDED = new Builder("Client.CachePagesDiscarded") .setDescription("Total number of pages discarded when restoring the page store.") @@ -2454,6 +2478,12 @@ public static String getSyncMetricName(long mountId) { .setMetricType(MetricType.GAUGE) .setIsClusterAggregated(false) .build(); + public static final MetricKey CLIENT_CACHE_HIT_REQUESTS = + new Builder("Client.CacheHitRequests") + .setDescription("Total number of requests of hitting the cache.") + .setMetricType(MetricType.COUNTER) + .setIsClusterAggregated(false) + .build(); public static final MetricKey CLIENT_CACHE_SPACE_AVAILABLE = new Builder("Client.CacheSpaceAvailable") .setDescription("Amount of bytes available in the client cache.") diff --git a/core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java b/core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java new file mode 100644 index 000000000000..12d8996e7370 --- /dev/null +++ b/core/common/src/main/java/alluxio/metrics/MultiDimensionalMetricsSystem.java @@ -0,0 +1,311 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.metrics; + +import alluxio.conf.Configuration; +import alluxio.conf.PropertyKey; +import alluxio.util.CommonUtils; +import alluxio.util.FormatUtils; + +import io.prometheus.metrics.config.PrometheusProperties; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.core.metrics.GaugeWithCallback; +import io.prometheus.metrics.core.metrics.Histogram; +import io.prometheus.metrics.exporter.common.PrometheusHttpRequest; +import io.prometheus.metrics.exporter.common.PrometheusHttpResponse; +import io.prometheus.metrics.exporter.common.PrometheusScrapeHandler; +import io.prometheus.metrics.exporter.servlet.jakarta.HttpExchangeAdapter; +import io.prometheus.metrics.instrumentation.jvm.JvmMetrics; +import io.prometheus.metrics.model.registry.PrometheusRegistry; +import io.prometheus.metrics.model.snapshots.Unit; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Enumeration; +import java.util.List; +import java.util.function.DoubleSupplier; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * A metrics system for all processes to define and collect all the metrics, + * and expose all the metrics to the web server. + */ +public final class MultiDimensionalMetricsSystem { + public static final Histogram DATA_ACCESS = Histogram.builder() + .name("alluxio_data_access") + .help("aggregated throughput of all the data access") + .unit(Unit.BYTES) + .labelNames("method") + .build(); + + public static final Counter META_OPERATION = Counter.builder() + .name("alluxio_meta_operation") + .help("counter of rpc calls of the meta operations") + .labelNames("op") + .build(); + + public static final Counter UFS_DATA_ACCESS = Counter.builder() + .name("alluxio_ufs_data_access") + .help("amount of the ufs access") + .unit(Unit.BYTES) + .labelNames("method") + .build(); + + public static final Counter CACHED_DATA_READ = Counter.builder() + .name("alluxio_cached_data_read") + .help("amount of the read cached data") + .unit(Unit.BYTES) + .build(); + + public static final Counter EXTERNAL_DATA_READ = Counter.builder() + .name("alluxio_external_data_read") + .help("amount of the read data when cache missed on client") + .unit(Unit.BYTES) + .build(); + + public static final Counter CACHED_EVICTED_DATA = Counter.builder() + .name("alluxio_cached_evicted_data") + .help("amount of the evicted data") + .unit(Unit.BYTES) + .build(); + + public static final DoubleSupplier NULL_SUPPLIER = () -> 0; + private static DoubleSupplier sCacheStorageSupplier = NULL_SUPPLIER; + private static DoubleSupplier sCacheManagerLoadingPagesSupplier = NULL_SUPPLIER; + + public static final GaugeWithCallback CACHED_STORAGE = GaugeWithCallback.builder() + .name("alluxio_cached_storage") + .help("amount of the cached data") + .unit(Unit.BYTES) + .callback(callback -> callback.call(sCacheStorageSupplier.getAsDouble())) + .build(); + + public static final GaugeWithCallback CACHED_CAPACITY = GaugeWithCallback.builder() + .name("alluxio_cached_capacity") + .help("configured maximum cache storage") + .unit(Unit.BYTES) + .callback(callback -> { + List sizes = Configuration.global().getList(PropertyKey.WORKER_PAGE_STORE_SIZES); + long sum = sizes.stream().map(FormatUtils::parseSpaceSize).reduce(0L, Long::sum); + callback.call(sum); + }) + .build(); + public static final GaugeWithCallback CACHE_MANAGER_LOADING_PAGES = GaugeWithCallback.builder() + .name("alluxio_cached_manager_loading_pages") + .help("number of pages being loaded from UFS into the cache manager") + .unit(Unit.BYTES) + .callback(callback -> callback.call(sCacheManagerLoadingPagesSupplier.getAsDouble())) + .build(); + + // Distributed load related + public static final Counter DISTRIBUTED_LOAD_JOB_FAILURE = Counter.builder() + .name("alluxio_distributed_load_job_failure") + .help("counter of the distributed load failure received on master") + .labelNames("reason", "final_attempt", "worker") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_SCANNED = Counter.builder() + .name("alluxio_distributed_load_job_scanned") + .help("counter of the inodes scanned in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_PROCESSED = Counter.builder() + .name("alluxio_distributed_load_job_processed") + .help("counter of the non empty file copies loaded in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_SKIPPED = Counter.builder() + .name("alluxio_distributed_load_job_skipped") + .help("counter of the inodes skipped in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_LOADED_BYTES = Counter.builder() + .name("alluxio_distributed_load_job_loaded_bytes") + .help("counter of the bytes loaded in distributed load") + .register(); + public static final Counter DISTRIBUTED_LOAD_JOB_DISPATCHED_SIZE = Counter.builder() + .name("alluxio_distributed_load_job_dispatched_size") + .help("distributed_load_job_batch_size") + .labelNames("worker") + .register(); + public static final Counter DISTRIBUTED_LOAD_WORKER_SUBTASKS_TOTAL = Counter.builder() + .name("alluxio_distributed_worker_subtasks_total") + .help("counter of the total subtasks a worker processes") + .register(); + public static final Counter DISTRIBUTED_LOAD_WORKER_BYTES_LOADED_TOTAL = Counter.builder() + .name("alluxio_distributed_worker_bytes_loaded_total") + .unit(Unit.BYTES) + .help("counter of the total bytes loaded in distributed load of a worker") + .register(); + public static final Counter DISTRIBUTED_LOAD_WORKER_ERRORS_TOTAL = Counter.builder() + .name("alluxio_distributed_worker_errors_total") + .help("counter of the total errors in distributed load of a worker") + .register(); + + // Client fallback related + public static final Counter CLIENT_RPC_RETRY_ON_DIFFERENT_WORKERS = Counter.builder() + .name("client_rpc_retry_on_different_workers") + .help("counter of client retry on different workers if multi replica is enabled") + .labelNames("op", "retry_count") + .build(); + + /** + * Initialize all the metrics. + */ + public static void initMetrics() { + JvmMetrics.builder().register(); + if (CommonUtils.PROCESS_TYPE.get() != CommonUtils.ProcessType.WORKER + && CommonUtils.PROCESS_TYPE.get() != CommonUtils.ProcessType.CLIENT) { + return; + } + if (CommonUtils.PROCESS_TYPE.get() == CommonUtils.ProcessType.CLIENT) { + PrometheusRegistry.defaultRegistry.register(EXTERNAL_DATA_READ); + } + PrometheusRegistry.defaultRegistry.register(DATA_ACCESS); + PrometheusRegistry.defaultRegistry.register(UFS_DATA_ACCESS); + PrometheusRegistry.defaultRegistry.register(META_OPERATION); + PrometheusRegistry.defaultRegistry.register(CACHED_DATA_READ); + PrometheusRegistry.defaultRegistry.register(CACHED_EVICTED_DATA); + PrometheusRegistry.defaultRegistry.register(CACHED_STORAGE); + PrometheusRegistry.defaultRegistry.register(CACHED_CAPACITY); + PrometheusRegistry.defaultRegistry.register(CACHE_MANAGER_LOADING_PAGES); + } + + /** + * Set the supplier for CACHE_STORAGE metrics. + * + * @param supplier supplier for cache storage + */ + public static void setCacheStorageSupplier(DoubleSupplier supplier) { + sCacheStorageSupplier = supplier; + } + + /** + * @param supplier the supplier for CACHE_MANAGER_LOADING_PAGES + */ + public static void setCacheManagerLoadingPagesSupplier( + DoubleSupplier supplier) { + MultiDimensionalMetricsSystem.sCacheManagerLoadingPagesSupplier = supplier; + } + + /** + * A servlet that exposes metrics data in prometheus format by HTTP. + */ + public static class WebHandler extends HttpServlet { + private static final long serialVersionUID = 1; + private static final String SERVLET_PATH = "/metrics"; + + private final PrometheusScrapeHandler mHandler = new PrometheusScrapeHandler( + PrometheusProperties.get(), PrometheusRegistry.defaultRegistry); + + /** + * Returns the servlet handler for Prometheus exposure. + * + * @return servlet handler + */ + public static ServletContextHandler getHandler() { + ServletContextHandler contextHandler = new ServletContextHandler(); + contextHandler.setContextPath(SERVLET_PATH); + contextHandler.addServlet(new ServletHolder(new WebHandler()), "/"); + return contextHandler; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { + mHandler.handleRequest(new ServletAdapter(req, resp)); + } + } + + /** + * An adapter to make the javax.servlet and jakarta.servlet compatible. + */ + private static class ServletAdapter extends HttpExchangeAdapter { + private final Request mRequest; + private final Response mResponse; + + public ServletAdapter(HttpServletRequest request, HttpServletResponse response) { + super(null, null); + mRequest = new Request(request); + mResponse = new Response(response); + } + + @Override + public PrometheusHttpRequest getRequest() { + return mRequest; + } + + @Override + public PrometheusHttpResponse getResponse() { + return mResponse; + } + + @Override + public void handleException(IOException e) throws IOException { + throw e; + } + + @Override + public void handleException(RuntimeException e) { + throw e; + } + + @Override + public void close() {} + + private static class Request implements PrometheusHttpRequest { + private final HttpServletRequest mRequest; + + public Request(HttpServletRequest request) { + mRequest = request; + } + + @Override + public String getQueryString() { + return mRequest.getQueryString(); + } + + @Override + public Enumeration getHeaders(String name) { + return mRequest.getHeaders(name); + } + + @Override + public String getMethod() { + return mRequest.getMethod(); + } + } + + private static class Response implements PrometheusHttpResponse { + private final HttpServletResponse mResponse; + + public Response(HttpServletResponse response) { + mResponse = response; + } + + @Override + public void setHeader(String name, String value) { + mResponse.setHeader(name, value); + } + + @Override + public OutputStream sendHeadersAndGetBody( + int statusCode, int contentLength) throws IOException { + if (mResponse.getHeader("Content-Length") == null && contentLength > 0) { + mResponse.setContentLength(contentLength); + } + + mResponse.setStatus(statusCode); + return mResponse.getOutputStream(); + } + } + } +} diff --git a/core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java b/core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java new file mode 100644 index 000000000000..5a0cab898826 --- /dev/null +++ b/core/common/src/main/java/alluxio/network/protocol/databuffer/DataFileChannel.java @@ -0,0 +1,86 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.network.protocol.databuffer; + +import com.google.common.base.Preconditions; +import io.netty.channel.DefaultFileRegion; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +/** + * A DataBuffer with the underlying data being a {@link FileChannel}. + */ +public final class DataFileChannel implements DataBuffer { + private final File mFile; + private final long mOffset; + private final long mLength; + + /** + * + * @param file The file + * @param offset The offset into the FileChannel + * @param length The length of the data to read + */ + public DataFileChannel(File file, long offset, long length) { + mFile = Preconditions.checkNotNull(file, "file"); + mOffset = offset; + mLength = length; + } + + @Override + public Object getNettyOutput() { + return new DefaultFileRegion(mFile, mOffset, mLength); + } + + @Override + public long getLength() { + return mLength; + } + + @Override + public ByteBuffer getReadOnlyByteBuffer() { + throw new UnsupportedOperationException( + "DataFileChannel#getReadOnlyByteBuffer is not implemented."); + } + + @Override + public void readBytes(byte[] dst, int dstIndex, int length) { + throw new UnsupportedOperationException("DataFileChannel#readBytes is not implemented."); + } + + @Override + public void readBytes(OutputStream outputStream, int length) throws IOException { + throw new UnsupportedOperationException("DataFileChannel#readBytes is not implemented."); + } + + @Override + public void readBytes(ByteBuffer outputBuf) { + throw new UnsupportedOperationException("DataFileChannel#readBytes is not implemented."); + } + + @Override + public int readableBytes() { + int lengthInt = (int) mLength; + Preconditions.checkArgument(mLength == (long) lengthInt, + "size of file %s is %s, cannot be cast to int", mFile, mLength); + return lengthInt; + } + + @Override + public void release() { + // Nothing we need to release explicitly, let GC take care of all objects. + } +} diff --git a/core/common/src/main/java/alluxio/util/io/ChannelAdapters.java b/core/common/src/main/java/alluxio/util/io/ChannelAdapters.java new file mode 100644 index 000000000000..d28a51fd64eb --- /dev/null +++ b/core/common/src/main/java/alluxio/util/io/ChannelAdapters.java @@ -0,0 +1,141 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.util.io; + +import io.netty.buffer.ByteBuf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.WritableByteChannel; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Channel adapters. + *
    + *

    Thread Safety

    + * The channel implementations in this class are thread safe, as per the general requirement + * of the {@link java.nio.channels.Channel} interface. However, in case a channel is based on + * an underlying buffer, cautions must be taken to preserve thread safety: + *
      + *
    • Two channels should not share the same underlying buffer and be operated on + * concurrently.
      • + *
    • A buffer should not be manipulated directly while there is an outstanding channel that + * uses it as the underlying buffer.
      • + *
    + */ +public class ChannelAdapters { + private ChannelAdapters() { } // utility class + + /** + * Adapts a ByteBuffer to be the destination of a {@link WritableByteChannel}. + *
    + * The returned channel, when its {@link WritableByteChannel#write(ByteBuffer)} method is called, + * will copy {@code p} bytes from the source buffer to the output buffer, + * where {@code p = min(src.remaining(), output.remaining())}. After the write is successful, + * the positions of both the source and output buffer will have increased by {@code p}. + * If the output buffer has no space remaining, while the source + * buffer has available bytes, {@link ClosedChannelException} is thrown. + * + * @param output the output byte buffer + * @return channel + */ + public static WritableByteChannel intoByteBuffer(ByteBuffer output) { + return new ByteBufferWritableChannel(output); + } + + /** + * Adapts a byte array to be the destination of a {@link WritableByteChannel}. + * Bytes will be written from the beginning of the array and to the end. + * + * @param output output buffer + * @return channel + */ + public static WritableByteChannel intoByteArray(byte[] output) { + return intoByteBuffer(ByteBuffer.wrap(output)); + } + + /** + * Adapts a byte array to be the destination of a {@link WritableByteChannel}. + * Bytes will be written from index {@code offset} to {@code offset + length}. + * + * @param output output buffer + * @param offset offset within the buffer + * @param length length of the writable region within the buffer + * @return channel + */ + public static WritableByteChannel intoByteArray(byte[] output, int offset, int length) { + return intoByteBuffer(ByteBuffer.wrap(output, offset, length)); + } + + /** + * todo(bowen): implement. + * @param output output + * @return channel + */ + public static WritableByteChannel intoByteBuf(ByteBuf output) { + throw new UnsupportedOperationException("todo"); + } + + @ThreadSafe + static class ByteBufferWritableChannel implements WritableByteChannel { + @GuardedBy("this") + private final ByteBuffer mOutput; + private boolean mClosed = false; + + ByteBufferWritableChannel(ByteBuffer byteBuffer) { + mOutput = byteBuffer; + } + + /** + * Copies bytes from the source buffer into the channel. + * + * @param src the buffer from which bytes are to be retrieved + * @return number of bytes written, 0 when the source buffer has no bytes remaining + * @throws ClosedChannelException if the channel has been explicitly closed, + * or when the output buffer does not have remaining space, + * but the source buffer has remaining readable bytes + */ + @Override + public int write(ByteBuffer src) throws IOException { + if (mClosed) { + throw new ClosedChannelException(); + } + synchronized (this) { + int srcRemaining = src.remaining(); + int outputRemaining = mOutput.remaining(); + if (outputRemaining == 0 && srcRemaining > 0) { + throw new ClosedChannelException(); + } + int bytesToCopy = Math.min(srcRemaining, outputRemaining); + ByteBuffer slice = src.slice(); + slice.limit(bytesToCopy); + mOutput.put(slice); + src.position(src.position() + bytesToCopy); + return bytesToCopy; + } + } + + @Override + public boolean isOpen() { + return !mClosed; + } + + @Override + public void close() throws IOException { + if (!mClosed) { + mClosed = true; + } + } + } +} diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java index 1564190d76b3..f1e9d1908744 100644 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java +++ b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockMetaStore.java @@ -11,6 +11,7 @@ package alluxio.worker.page; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.DefaultPageMetaStore; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; @@ -21,6 +22,7 @@ import alluxio.client.quota.CacheScope; import alluxio.collections.IndexDefinition; import alluxio.collections.IndexedSet; +import alluxio.exception.FileDoesNotExistException; import alluxio.exception.PageNotFoundException; import alluxio.exception.runtime.BlockDoesNotExistRuntimeException; import alluxio.worker.block.BlockStoreEventListener; @@ -217,21 +219,6 @@ public void addPage(PageId pageId, PageInfo pageInfo) { mDelegate.addPage(pageId, pageInfo); } - /** - * Gets the block meta for a page of the block. - * @param pageId the page ID - * @return block meta - * @throws BlockDoesNotExistRuntimeException when the block is not being stored in the store - */ - private PagedBlockMeta getBlockMetaOfPage(PageId pageId) { - long blockId = BlockPageId.downcast(pageId).getBlockId(); - PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); - if (blockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - return blockMeta; - } - @Override @GuardedBy("getLock().writeLock()") public void addTempPage(PageId pageId, PageInfo pageInfo) { @@ -249,34 +236,19 @@ public void commitFile(String fileId, String newFileId) throws PageNotFoundExcep mDelegate.commitFile(fileId, newFileId); } - /** - * @param blockId - * @return the permanent block meta after committing - */ - public PagedBlockMeta commit(long blockId) { - PagedTempBlockMeta tempBlockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); - if (tempBlockMeta == null) { - throw new BlockDoesNotExistRuntimeException(blockId); - } - PagedBlockMeta blockMeta = new PagedBlockMeta(tempBlockMeta.getBlockId(), - tempBlockMeta.getBlockSize(), tempBlockMeta.getDir()); - mTempBlocks.remove(tempBlockMeta); - mBlocks.add(blockMeta); - try { - commitFile(BlockPageId.tempFileIdOf(blockId), - BlockPageId.fileIdOf(blockId, blockMeta.getBlockSize())); - } catch (PageNotFoundException e) { - // this should be unreachable, since we have checked the existence of the block - // otherwise it's a bug - LOG.error("Cannot commit block {} as no pages are found", blockId, e); - throw new RuntimeException(e); - } - return blockMeta; + @Override + public PageStoreDir getStoreDirOfFile(String fileId) throws FileDoesNotExistException { + return null; } @Override - public List getStoreDirs() { - return mDelegate.getStoreDirs(); + public Optional getUsage() { + return Optional.empty(); + } + + @Override + public PageInfo removePage(PageId pageId, boolean isTemporary) throws PageNotFoundException { + return null; } @Override @@ -298,6 +270,11 @@ public PageInfo removePage(PageId pageId) throws PageNotFoundException { return pageInfo; } + @Override + public List getStoreDirs() { + return mDelegate.getStoreDirs(); + } + @Override @GuardedBy("getLock().readLock()") public long bytes() { @@ -317,6 +294,11 @@ public void reset() { mBlocks.clear(); } + @Override + public Set getAllPagesByFileId(String fileId) { + return null; + } + @Override @GuardedBy("getLock().readLock()") public PageInfo evict(CacheScope cacheScope, PageStoreDir pageStoreDir) { @@ -358,6 +340,31 @@ public void registerBlockStoreEventListener(BlockStoreEventListener listener) { mBlockStoreEventListeners.add(listener); } + /** + * @param blockId + * @return the permanent block meta after committing + */ + public PagedBlockMeta commit(long blockId) { + PagedTempBlockMeta tempBlockMeta = mTempBlocks.getFirstByField(INDEX_TEMP_BLOCK_ID, blockId); + if (tempBlockMeta == null) { + throw new BlockDoesNotExistRuntimeException(blockId); + } + PagedBlockMeta blockMeta = new PagedBlockMeta(tempBlockMeta.getBlockId(), + tempBlockMeta.getBlockSize(), tempBlockMeta.getDir()); + mTempBlocks.remove(tempBlockMeta); + mBlocks.add(blockMeta); + try { + commitFile(BlockPageId.tempFileIdOf(blockId), + BlockPageId.fileIdOf(blockId, blockMeta.getBlockSize())); + } catch (PageNotFoundException e) { + // this should be unreachable, since we have checked the existence of the block + // otherwise it's a bug + LOG.error("Cannot commit block {} as no pages are found", blockId, e); + throw new RuntimeException(e); + } + return blockMeta; + } + /** * @return brief store meta */ @@ -412,4 +419,19 @@ private static PagedBlockStoreDir downcast(PageStoreDir pageStoreDir) { String.format("Unexpected page store dir type %s, for worker page store it should be %s", pageStoreDir.getClass().getSimpleName(), PagedBlockStoreDir.class.getSimpleName())); } + + /** + * Gets the block meta for a page of the block. + * @param pageId the page ID + * @return block meta + * @throws BlockDoesNotExistRuntimeException when the block is not being stored in the store + */ + private PagedBlockMeta getBlockMetaOfPage(PageId pageId) { + long blockId = BlockPageId.downcast(pageId).getBlockId(); + PagedBlockMeta blockMeta = mBlocks.getFirstByField(INDEX_BLOCK_ID, blockId); + if (blockMeta == null) { + throw new BlockDoesNotExistRuntimeException(blockId); + } + return blockMeta; + } } diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java index 41b7a95d4016..69c12528caa6 100644 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java +++ b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockReader.java @@ -14,8 +14,9 @@ import alluxio.client.file.CacheContext; import alluxio.client.file.cache.CacheManager; import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.store.PageReadTargetBuffer; import alluxio.exception.runtime.AlluxioRuntimeException; +import alluxio.file.NettyBufTargetBuffer; +import alluxio.file.ReadTargetBuffer; import alluxio.grpc.ErrorType; import alluxio.metrics.MetricKey; import alluxio.metrics.MetricsSystem; @@ -105,7 +106,7 @@ private long read(ByteBuf byteBuf, long offset, long length) throws IOException Preconditions.checkArgument(byteBuf.writableBytes() >= length, "buffer overflow, trying to write %s bytes, only %s writable", length, byteBuf.writableBytes()); - PageReadTargetBuffer target = new NettyBufTargetBuffer(byteBuf); + ReadTargetBuffer target = new NettyBufTargetBuffer(byteBuf); long bytesRead = 0; while (bytesRead < length) { long pos = offset + bytesRead; diff --git a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java index b5d232651200..e26a683a6be0 100644 --- a/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java +++ b/core/server/worker/src/main/java/alluxio/worker/page/PagedBlockStoreDir.java @@ -14,6 +14,7 @@ import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_MEDIUM; import static alluxio.worker.page.PagedBlockStoreMeta.DEFAULT_TIER; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; import alluxio.client.file.cache.PageInfo; import alluxio.client.file.cache.PageStore; @@ -171,6 +172,10 @@ public boolean reserve(long bytes) { return mDelegate.reserve(bytes); } + @Override + public void deleteTempPage(PageInfo bytes) { + } + @Override public long deletePage(PageInfo pageInfo) { long blockId = BlockPageId.downcast(pageInfo.getPageId()).getBlockId(); @@ -277,4 +282,9 @@ public Set getBlockPages(long blockId) { return mBlockToPagesMap.get(blockId).stream().map(PageInfo::getPageId) .collect(Collectors.toSet()); } + + @Override + public Optional getUsage() { + return Optional.empty(); + } } diff --git a/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java b/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java index 2c21647ec766..b7aa85325f9b 100644 --- a/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java +++ b/core/server/worker/src/test/java/alluxio/worker/page/ByteArrayCacheManager.java @@ -13,12 +13,16 @@ import alluxio.client.file.CacheContext; import alluxio.client.file.cache.CacheManager; +import alluxio.client.file.cache.CacheUsage; import alluxio.client.file.cache.PageId; -import alluxio.client.file.cache.store.PageReadTargetBuffer; +import alluxio.file.ReadTargetBuffer; +import alluxio.network.protocol.databuffer.DataFileChannel; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; /** * Implementation of cache manager that stores cached data in byte arrays in memory. @@ -36,6 +40,11 @@ class ByteArrayCacheManager implements CacheManager { mPages = new HashMap<>(); } + @Override + public void commitFile(String fileId) { + throw new UnsupportedOperationException("commitFile method is unsupported. "); + } + @Override public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { byte[] data = new byte[page.remaining()]; @@ -46,7 +55,7 @@ public boolean put(PageId pageId, ByteBuffer page, CacheContext cacheContext) { } @Override - public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuffer target, + public int get(PageId pageId, int pageOffset, int bytesToRead, ReadTargetBuffer target, CacheContext cacheContext) { if (!mPages.containsKey(pageId)) { return 0; @@ -56,6 +65,24 @@ public int get(PageId pageId, int pageOffset, int bytesToRead, PageReadTargetBuf return bytesToRead; } + @Override + public int getAndLoad(PageId pageId, int pageOffset, int bytesToRead, + ReadTargetBuffer buffer, CacheContext cacheContext, + Supplier externalDataSupplier) { + int bytesRead = get(pageId, pageOffset, + bytesToRead, buffer, cacheContext); + if (bytesRead > 0) { + return bytesRead; + } + byte[] page = externalDataSupplier.get(); + if (page.length == 0) { + return 0; + } + buffer.writeBytes(page, pageOffset, bytesToRead); + put(pageId, page, cacheContext); + return bytesToRead; + } + @Override public boolean delete(PageId pageId) { return mPages.remove(pageId) != null; @@ -72,7 +99,50 @@ public boolean append(PageId pageId, int appendAt, byte[] page, CacheContext cac } @Override - public void close() throws Exception { + public void deleteFile(String fileId) { + mPages.keySet().removeIf(pageId -> pageId.getFileId().equals(fileId)); + } + + @Override + public void deleteTempFile(String fileId) { + mPages.keySet().removeIf(pageId -> pageId.getFileId().equals(fileId)); + } + + @Override + public Optional getUsage() { + return Optional.of(new Usage()); + } + + @Override + public Optional getDataFileChannel(PageId pageId, int pageOffset, + int bytesToRead, CacheContext cacheContext) { + return Optional.empty(); + } + + class Usage implements CacheUsage { + @Override + public Optional partitionedBy(PartitionDescriptor partition) { + return Optional.empty(); + } + + @Override + public long used() { + return mPages.values().stream().mapToInt(page -> page.length).sum(); + } + + @Override + public long available() { + return Integer.MAX_VALUE; + } + + @Override + public long capacity() { + return Integer.MAX_VALUE; + } + } + + @Override + public void close() { // no-op } } diff --git a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java index fbbc687a5105..f13ba92abbfb 100644 --- a/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java +++ b/core/server/worker/src/test/java/alluxio/worker/page/PagedBlockWriterTest.java @@ -25,12 +25,12 @@ import alluxio.client.file.cache.PageStore; import alluxio.client.file.cache.evictor.CacheEvictor; import alluxio.client.file.cache.evictor.FIFOCacheEvictor; -import alluxio.client.file.cache.store.ByteArrayTargetBuffer; import alluxio.client.file.cache.store.LocalPageStoreDir; import alluxio.client.file.cache.store.PageStoreOptions; import alluxio.conf.Configuration; import alluxio.conf.InstancedConfiguration; import alluxio.conf.PropertyKey; +import alluxio.file.ByteArrayTargetBuffer; import alluxio.util.CommonUtils; import alluxio.util.WaitForOptions; import alluxio.util.io.BufferUtils; diff --git a/pom.xml b/pom.xml index 9a42a65aaffa..f5caf2b4ab5c 100644 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,7 @@ 1.6.3 5.3.5 2.0.7 + 1.0.0 0.8.0 31.0.1-jre 1.11.0 @@ -406,6 +407,21 @@ netty-transport-native-unix-common ${netty.version}     + + io.prometheus + prometheus-metrics-core + ${prometheus.client.version} + + + io.prometheus + prometheus-metrics-exporter-servlet-jakarta + ${prometheus.client.version} + + + io.prometheus + prometheus-metrics-instrumentation-jvm + ${prometheus.client.version} + io.prometheus simpleclient @@ -426,6 +442,11 @@ swagger-annotations 1.6.2 + + jakarta.servlet + jakarta.servlet-api + 5.0.0 + jakarta.ws.rs jakarta.ws.rs-api diff --git a/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java b/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java index 49ac00cd88b3..3149a7bdf979 100644 --- a/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java +++ b/tests/src/test/java/alluxio/client/fs/LocalCacheManagerIntegrationTest.java @@ -13,6 +13,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import alluxio.Constants; @@ -84,12 +85,6 @@ public void after() throws Exception { } } - @Test - public void newCacheRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testNewCache(); - } - @Test public void newCacheLocal() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -111,12 +106,6 @@ private void testPageCached(PageId pageId) { assertArrayEquals(PAGE, mBuffer); } - @Test - public void loadCacheRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testLoadCache(); - } - @Test public void loadCacheLocal() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -172,25 +161,6 @@ public void loadCacheMismatchedPageSize() throws Exception { testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_PAGE_SIZE, PAGE_SIZE_BYTES * 2); } - @Test - public void loadCacheMismatchedStoreTypeRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); - testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - } - - @Test - public void loadCacheMismatchedStoreTypeLocal() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); - } - - @Test - public void loadCacheSmallerNewCacheSizeRocks() throws Exception { - mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.ROCKS); - testLoadCacheConfMismatch(PropertyKey.USER_CLIENT_CACHE_SIZE, - String.valueOf(CACHE_SIZE_BYTES / 2)); - } - @Test public void loadCacheSmallerNewCacheSizeLocal() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -215,6 +185,13 @@ public void loadCacheSmallerNewCacheSizeLocal() throws Exception { } } + /** + * Test the case that cache manager will properly handle invalid page file. + * if there is an invalid page file in the cache dir, cache manage will not recognize such + * file and will delete this file. Other valid page files are not affected + * and the cache manager should be able to read data from them normally. + * @throws Exception + */ @Test public void loadCacheWithInvalidPageFile() throws Exception { mConf.set(PropertyKey.USER_CLIENT_CACHE_STORE_TYPE, PageStoreType.LOCAL); @@ -222,9 +199,14 @@ public void loadCacheWithInvalidPageFile() throws Exception { mCacheManager.close(); // creates with an invalid page file stored String rootDir = mPageMetaStore.getStoreDirs().get(0).getRootPath().toString(); - FileUtils.createFile(Paths.get(rootDir, "invalidPageFile").toString()); + String invalidPageFileName = Paths.get(rootDir, "invalidPageFile").toString(); + FileUtils.createFile(invalidPageFileName); mCacheManager = LocalCacheManager.create(mCacheManagerOptions, mPageMetaStore); - assertEquals(0, mCacheManager.get(PAGE_ID, PAGE_SIZE_BYTES, mBuffer, 0)); + // There is an invalid file in the cache dir. But the cache manager will not recognize it as a + // valid page file and will delete it, and then will continue starting as normal. + assertEquals(PAGE_SIZE_BYTES, mCacheManager.get(PAGE_ID, PAGE_SIZE_BYTES, mBuffer, 0)); + assertArrayEquals(PAGE, mBuffer); + assertFalse(FileUtils.exists(invalidPageFileName)); } @Test