From 64cd7631c2b026dce3c43c18224ca346edabdea3 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Fri, 16 Dec 2016 00:18:37 +0800
Subject: [PATCH 01/17] Add a LocalFirstPolicy that without evict action
---
.../java/alluxio/client/block/BlockWorkerInfo.java | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java b/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
index ef4a249839c6..e01e9ba92199 100644
--- a/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
+++ b/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
@@ -11,6 +11,8 @@
package alluxio.client.block;
+import alluxio.Configuration;
+import alluxio.PropertyKey;
import alluxio.annotation.PublicApi;
import alluxio.wire.WorkerNetAddress;
@@ -27,6 +29,8 @@ public final class BlockWorkerInfo {
private final WorkerNetAddress mNetAddress;
private final long mCapacityBytes;
private final long mUsedBytes;
+ private long mBlockSizeBytes;
+ private long mUserFileWriteCapacityReservedRatio;
/**
* Constructs the block worker information.
@@ -55,6 +59,16 @@ public long getCapacityBytes() {
return mCapacityBytes;
}
+ /**
+ * @return the available bytes of the worker
+ */
+ public long getAvailableBytes() {
+ mBlockSizeBytes = Configuration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT);
+ mUserFileWriteCapacityReservedRatio = Configuration
+ .getBytes(PropertyKey.USER_FILE_WRITE_CAPACITY_RESERVED_RATIO);
+ return mCapacityBytes - mUsedBytes - (mUserFileWriteCapacityReservedRatio * mBlockSizeBytes);
+ }
+
/**
* @return the used bytes of the worker
*/
From 3bdc29ea47e752957437e3ca97d2d9d8be7450ea Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Fri, 16 Dec 2016 00:19:04 +0800
Subject: [PATCH 02/17] Add a LocalFirstPolicy that without evict action
---
.../LocalFirstWithoutEvictionPolicy.java | 88 +++++++++++++++++++
1 file changed, 88 insertions(+)
create mode 100644 core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java
new file mode 100644
index 000000000000..dfb5b2114a8f
--- /dev/null
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java
@@ -0,0 +1,88 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.client.file.policy;
+
+import alluxio.client.block.BlockWorkerInfo;
+import alluxio.util.network.NetworkAddressUtils;
+import alluxio.wire.WorkerNetAddress;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+
+import java.util.Collections;
+import java.util.List;
+
+import javax.annotation.concurrent.ThreadSafe;
+/**
+ * A policy that returns local host first, and if the local worker doesn't have enough availability,
+ * it randomly picks a worker from the active workers list for each block write.
+ * If No worker meets the demands, return local host.
+ */
+@ThreadSafe
+public class LocalFirstWithoutEvictionPolicy implements FileWriteLocationPolicy {
+ private String mLocalHostName;
+ /**
+ * Constructs a {@link LocalFirstWithoutEvictionPolicy}.
+ */
+ public LocalFirstWithoutEvictionPolicy() {
+ mLocalHostName = NetworkAddressUtils.getLocalHostName();
+ }
+
+ @Override
+ public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList,
+ long blockSizeBytes) {
+ // try the local host first
+ WorkerNetAddress localWorkerNetAddress = null;
+ for (BlockWorkerInfo workerInfo : workerInfoList) {
+ if (workerInfo.getNetAddress().getHost().equals(mLocalHostName)) {
+ localWorkerNetAddress = workerInfo.getNetAddress();
+ if (workerInfo.getAvailableBytes() >= blockSizeBytes) {
+ return localWorkerNetAddress;
+ }
+ }
+ }
+
+ // otherwise randomly pick a worker that has enough availability
+ List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList);
+ Collections.shuffle(shuffledWorkers);
+ for (BlockWorkerInfo workerInfo : shuffledWorkers) {
+ if (workerInfo.getAvailableBytes() >= blockSizeBytes) {
+ return workerInfo.getNetAddress();
+ }
+ }
+ return localWorkerNetAddress;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof LocalFirstWithoutEvictionPolicy)) {
+ return false;
+ }
+ LocalFirstWithoutEvictionPolicy that = (LocalFirstWithoutEvictionPolicy) o;
+ return Objects.equal(mLocalHostName, that.mLocalHostName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(mLocalHostName);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("localHostName", mLocalHostName)
+ .toString();
+ }
+}
From 8be64bc099cb08ff446f21fe2ca920a54fdf8791 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Fri, 16 Dec 2016 00:19:42 +0800
Subject: [PATCH 03/17] add user file write capacity reserved ratio
---
core/common/src/main/java/alluxio/PropertyKey.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/core/common/src/main/java/alluxio/PropertyKey.java b/core/common/src/main/java/alluxio/PropertyKey.java
index 1576e52d1909..599baf5ca9a0 100644
--- a/core/common/src/main/java/alluxio/PropertyKey.java
+++ b/core/common/src/main/java/alluxio/PropertyKey.java
@@ -252,6 +252,7 @@ public enum PropertyKey {
Name.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS, 300 * Constants.SECOND_MS),
USER_FILE_WRITE_LOCATION_POLICY(Name.USER_FILE_WRITE_LOCATION_POLICY,
"alluxio.client.file.policy.LocalFirstPolicy"),
+ USER_FILE_WRITE_CAPACITY_RESERVED_RATIO(Name.USER_FILE_WRITE_CAPACITY_RESERVED_RATIO, 1),
USER_FILE_WRITE_TYPE_DEFAULT(Name.USER_FILE_WRITE_TYPE_DEFAULT, "MUST_CACHE"),
USER_HEARTBEAT_INTERVAL_MS(Name.USER_HEARTBEAT_INTERVAL_MS, 1000),
USER_LINEAGE_ENABLED(Name.USER_LINEAGE_ENABLED, false),
@@ -661,6 +662,8 @@ public static final class Name {
"alluxio.user.file.worker.client.pool.gc.threshold.ms";
public static final String USER_FILE_WRITE_LOCATION_POLICY =
"alluxio.user.file.write.location.policy.class";
+ public static final String USER_FILE_WRITE_CAPACITY_RESERVED_RATIO =
+ "alluxio.user.file.write.capacity.reserved.ratio";
public static final String USER_FILE_WRITE_TYPE_DEFAULT = "alluxio.user.file.writetype.default";
public static final String USER_HEARTBEAT_INTERVAL_MS = "alluxio.user.heartbeat.interval.ms";
public static final String USER_LINEAGE_ENABLED = "alluxio.user.lineage.enabled";
From 6bf0036fc7ec22ad37e5a65ff0ba8b1a0f1d23a7 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Fri, 16 Dec 2016 00:20:48 +0800
Subject: [PATCH 04/17] add describe of
alluxio.user.file.write.capacity.reserved.ratio
---
docs/_data/table/cn/user-configuration.yml | 2 ++
docs/_data/table/en/user-configuration.yml | 3 +++
docs/_data/table/user-configuration.csv | 1 +
3 files changed, 6 insertions(+)
diff --git a/docs/_data/table/cn/user-configuration.yml b/docs/_data/table/cn/user-configuration.yml
index 3ecc1214eea3..7db0ed6f9603 100644
--- a/docs/_data/table/cn/user-configuration.yml
+++ b/docs/_data/table/cn/user-configuration.yml
@@ -28,6 +28,8 @@ alluxio.user.file.worker.client.threads:
文件worker client从worker读取数据时使用的线程数目。
alluxio.user.file.write.location.policy.class:
选择worker进行写文件数据块时的默认定位机制。
+alluxio.user.file.write.capacity.reserved.ratio:
+ 当用户选择LocalFirstWithoutEvictionPolicy作为写文件数据块的定位机制时，使用文件块大小和这个参数的乘机来作为worker的预留存储空间。值为[0,1]。
alluxio.user.file.readtype.default:
创建Alluxio文件时的默认读类型。可选值为`CACHE_PROMOTE` (如果数据已经在Alluxio存储内，将其移动到最高存储层，如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层)、`CACHE` (如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层), `NO_CACHE` (数据不与Alluxio交互，如果是从Alluxio中进行读取，将不会发生数据块迁移或者剔除)。
alluxio.user.file.writetype.default:
diff --git a/docs/_data/table/en/user-configuration.yml b/docs/_data/table/en/user-configuration.yml
index 2d67e6b158eb..dfc06fcd448c 100644
--- a/docs/_data/table/en/user-configuration.yml
+++ b/docs/_data/table/en/user-configuration.yml
@@ -30,6 +30,9 @@ alluxio.user.file.worker.client.threads:
How many threads to use for file worker clients to read from workers.
alluxio.user.file.write.location.policy.class:
The default location policy for choosing workers for writing a file's blocks
+alluxio.user.file.write.capacity.reserved.ratio:
+ The portion of space reserced in worker when user use the LocalFirstWithoutEvictionPolicy class
+ as file write location policy (a value between 0 and 1).
alluxio.user.file.readtype.default:
Default read type when creating Alluxio files.
Valid options are `CACHE_PROMOTE` (move data to highest tier if already in Alluxio storage,
diff --git a/docs/_data/table/user-configuration.csv b/docs/_data/table/user-configuration.csv
index 3858054f6df5..8a9e44c271db 100644
--- a/docs/_data/table/user-configuration.csv
+++ b/docs/_data/table/user-configuration.csv
@@ -12,6 +12,7 @@ alluxio.user.file.master.client.threads,10
alluxio.user.file.waitcompleted.poll.ms,1000
alluxio.user.file.worker.client.threads,10
alluxio.user.file.write.location.policy.class,alluxio.client.file.policy.LocalFirstPolicy
+alluxio.user.file.write.capacity.reserved.ratio,1
alluxio.user.file.readtype.default,CACHE_PROMOTE
alluxio.user.file.writetype.default,MUST_CACHE
alluxio.user.heartbeat.interval.ms,1000
From 536e53074246ba82d64025ad659d9435377e21bd Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Fri, 16 Dec 2016 00:21:21 +0800
Subject: [PATCH 05/17] add LocalFirstWithoutEvictionPolicy test
---
.../LocalFirstWithoutEvictionPolicyTest.java | 85 +++++++++++++++++++
1 file changed, 85 insertions(+)
create mode 100644 core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java
diff --git a/core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java b/core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java
new file mode 100644
index 000000000000..7c79020d46bb
--- /dev/null
+++ b/core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java
@@ -0,0 +1,85 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.client.file.policy;
+
+import alluxio.CommonTestUtils;
+import alluxio.Constants;
+import alluxio.client.block.BlockWorkerInfo;
+import alluxio.util.network.NetworkAddressUtils;
+import alluxio.wire.WorkerNetAddress;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests {@link LocalFirstWithoutEvictionPolicy}.
+ */
+public class LocalFirstWithoutEvictionPolicyTest {
+ private static final int PORT = 1;
+
+ /**
+ * Tests that the local host is returned first.
+ */
+ @Test
+ public void getLocalFirst() {
+ String localhostName = NetworkAddressUtils.getLocalHostName();
+ LocalFirstWithoutEvictionPolicy policy = new LocalFirstWithoutEvictionPolicy();
+ List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
+ workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1")
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
+ workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName)
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
+ Assert.assertEquals(localhostName,
+ policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
+ }
+
+ /**
+ * Tests that another worker is picked in case the local host does not have enough space.
+ */
+ @Test
+ public void getOthersWhenNotEnoughSpaceOnLocal() {
+ String localhostName = NetworkAddressUtils.getLocalHostName();
+ LocalFirstWithoutEvictionPolicy policy = new LocalFirstWithoutEvictionPolicy();
+ List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
+ workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1")
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
+ workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName)
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.MB, Constants.MB));
+ Assert.assertEquals("worker1",
+ policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
+ }
+
+ /**
+ * Tests that local host is picked if none of the workers has enough availability.
+ */
+ @Test
+ public void getLocalWhenNoneHasSpace() {
+ String localhostName = NetworkAddressUtils.getLocalHostName();
+ LocalFirstWithoutEvictionPolicy policy = new LocalFirstWithoutEvictionPolicy();
+ List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
+ workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1")
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
+ workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName)
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, Constants.MB));
+ Assert.assertEquals(localhostName,
+ policy.getWorkerForNextBlock(workerInfoList, Constants.GB).getHost());
+ }
+
+ @Test
+ public void equalsTest() throws Exception {
+ CommonTestUtils.testEquals(LocalFirstWithoutEvictionPolicy.class);
+ }
+
+}
From f8516e6bb60d9e189d088014bb17376fea342090 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 12:17:23 +0800
Subject: [PATCH 06/17] Fixed few minor review comments
---
.../java/alluxio/client/block/BlockWorkerInfo.java | 10 ++++------
...icy.java => LocalFirstAvoidEvictionPolicy.java} | 10 +++++-----
...java => LocalFirstAvoidEvictionPolicyTest.java} | 14 +++++++-------
core/common/src/main/java/alluxio/PropertyKey.java | 6 +++---
docs/_data/table/cn/user-configuration.yml | 4 ++--
docs/_data/table/en/user-configuration.yml | 6 +++---
docs/_data/table/user-configuration.csv | 2 +-
7 files changed, 25 insertions(+), 27 deletions(-)
rename core/client/src/main/java/alluxio/client/file/policy/{LocalFirstWithoutEvictionPolicy.java => LocalFirstAvoidEvictionPolicy.java} (88%)
rename core/client/src/test/java/alluxio/client/file/policy/{LocalFirstWithoutEvictionPolicyTest.java => LocalFirstAvoidEvictionPolicyTest.java} (87%)
diff --git a/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java b/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
index e01e9ba92199..0a4d08481869 100644
--- a/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
+++ b/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
@@ -29,8 +29,7 @@ public final class BlockWorkerInfo {
private final WorkerNetAddress mNetAddress;
private final long mCapacityBytes;
private final long mUsedBytes;
- private long mBlockSizeBytes;
- private long mUserFileWriteCapacityReservedRatio;
+ private long mUserFileWriteCapacityReserved;
/**
* Constructs the block worker information.
@@ -63,10 +62,9 @@ public long getCapacityBytes() {
* @return the available bytes of the worker
*/
public long getAvailableBytes() {
- mBlockSizeBytes = Configuration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT);
- mUserFileWriteCapacityReservedRatio = Configuration
- .getBytes(PropertyKey.USER_FILE_WRITE_CAPACITY_RESERVED_RATIO);
- return mCapacityBytes - mUsedBytes - (mUserFileWriteCapacityReservedRatio * mBlockSizeBytes);
+ mUserFileWriteCapacityReserved = Configuration
+ .getBytes(PropertyKey.USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES);
+ return mCapacityBytes - mUsedBytes - mUserFileWriteCapacityReserved;
}
/**
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
similarity index 88%
rename from core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java
rename to core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index dfb5b2114a8f..6baaced4d10b 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -28,12 +28,12 @@
* If No worker meets the demands, return local host.
*/
@ThreadSafe
-public class LocalFirstWithoutEvictionPolicy implements FileWriteLocationPolicy {
+public class LocalFirstAvoidEvictionPolicy implements FileWriteLocationPolicy {
private String mLocalHostName;
/**
- * Constructs a {@link LocalFirstWithoutEvictionPolicy}.
+ * Constructs a {@link LocalFirstAvoidEvictionPolicy}.
*/
- public LocalFirstWithoutEvictionPolicy() {
+ public LocalFirstAvoidEvictionPolicy() {
mLocalHostName = NetworkAddressUtils.getLocalHostName();
}
@@ -67,10 +67,10 @@ public boolean equals(Object o) {
if (this == o) {
return true;
}
- if (!(o instanceof LocalFirstWithoutEvictionPolicy)) {
+ if (!(o instanceof LocalFirstAvoidEvictionPolicy)) {
return false;
}
- LocalFirstWithoutEvictionPolicy that = (LocalFirstWithoutEvictionPolicy) o;
+ LocalFirstAvoidEvictionPolicy that = (LocalFirstAvoidEvictionPolicy) o;
return Objects.equal(mLocalHostName, that.mLocalHostName);
}
diff --git a/core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java b/core/client/src/test/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicyTest.java
similarity index 87%
rename from core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java
rename to core/client/src/test/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicyTest.java
index 7c79020d46bb..f058fa89dca2 100644
--- a/core/client/src/test/java/alluxio/client/file/policy/LocalFirstWithoutEvictionPolicyTest.java
+++ b/core/client/src/test/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicyTest.java
@@ -24,9 +24,9 @@
import java.util.List;
/**
- * Tests {@link LocalFirstWithoutEvictionPolicy}.
+ * Tests {@link LocalFirstAvoidEvictionPolicy}.
*/
-public class LocalFirstWithoutEvictionPolicyTest {
+public class LocalFirstAvoidEvictionPolicyTest {
private static final int PORT = 1;
/**
@@ -35,7 +35,7 @@ public class LocalFirstWithoutEvictionPolicyTest {
@Test
public void getLocalFirst() {
String localhostName = NetworkAddressUtils.getLocalHostName();
- LocalFirstWithoutEvictionPolicy policy = new LocalFirstWithoutEvictionPolicy();
+ LocalFirstAvoidEvictionPolicy policy = new LocalFirstAvoidEvictionPolicy();
List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1")
.setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
@@ -51,7 +51,7 @@ public void getLocalFirst() {
@Test
public void getOthersWhenNotEnoughSpaceOnLocal() {
String localhostName = NetworkAddressUtils.getLocalHostName();
- LocalFirstWithoutEvictionPolicy policy = new LocalFirstWithoutEvictionPolicy();
+ LocalFirstAvoidEvictionPolicy policy = new LocalFirstAvoidEvictionPolicy();
List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1")
.setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
@@ -67,10 +67,10 @@ public void getOthersWhenNotEnoughSpaceOnLocal() {
@Test
public void getLocalWhenNoneHasSpace() {
String localhostName = NetworkAddressUtils.getLocalHostName();
- LocalFirstWithoutEvictionPolicy policy = new LocalFirstWithoutEvictionPolicy();
+ LocalFirstAvoidEvictionPolicy policy = new LocalFirstAvoidEvictionPolicy();
List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1")
- .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
+ .setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, Constants.MB));
workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName)
.setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, Constants.MB));
Assert.assertEquals(localhostName,
@@ -79,7 +79,7 @@ public void getLocalWhenNoneHasSpace() {
@Test
public void equalsTest() throws Exception {
- CommonTestUtils.testEquals(LocalFirstWithoutEvictionPolicy.class);
+ CommonTestUtils.testEquals(LocalFirstAvoidEvictionPolicy.class);
}
}
diff --git a/core/common/src/main/java/alluxio/PropertyKey.java b/core/common/src/main/java/alluxio/PropertyKey.java
index 599baf5ca9a0..869e3ec18f1e 100644
--- a/core/common/src/main/java/alluxio/PropertyKey.java
+++ b/core/common/src/main/java/alluxio/PropertyKey.java
@@ -252,7 +252,7 @@ public enum PropertyKey {
Name.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS, 300 * Constants.SECOND_MS),
USER_FILE_WRITE_LOCATION_POLICY(Name.USER_FILE_WRITE_LOCATION_POLICY,
"alluxio.client.file.policy.LocalFirstPolicy"),
- USER_FILE_WRITE_CAPACITY_RESERVED_RATIO(Name.USER_FILE_WRITE_CAPACITY_RESERVED_RATIO, 1),
+ USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES(Name.USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES, "0MB"),
USER_FILE_WRITE_TYPE_DEFAULT(Name.USER_FILE_WRITE_TYPE_DEFAULT, "MUST_CACHE"),
USER_HEARTBEAT_INTERVAL_MS(Name.USER_HEARTBEAT_INTERVAL_MS, 1000),
USER_LINEAGE_ENABLED(Name.USER_LINEAGE_ENABLED, false),
@@ -662,8 +662,8 @@ public static final class Name {
"alluxio.user.file.worker.client.pool.gc.threshold.ms";
public static final String USER_FILE_WRITE_LOCATION_POLICY =
"alluxio.user.file.write.location.policy.class";
- public static final String USER_FILE_WRITE_CAPACITY_RESERVED_RATIO =
- "alluxio.user.file.write.capacity.reserved.ratio";
+ public static final String USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES =
+ "alluxio.user.file.write.capacity.reserved.size.bytes";
public static final String USER_FILE_WRITE_TYPE_DEFAULT = "alluxio.user.file.writetype.default";
public static final String USER_HEARTBEAT_INTERVAL_MS = "alluxio.user.heartbeat.interval.ms";
public static final String USER_LINEAGE_ENABLED = "alluxio.user.lineage.enabled";
diff --git a/docs/_data/table/cn/user-configuration.yml b/docs/_data/table/cn/user-configuration.yml
index 7db0ed6f9603..2741f46e63ca 100644
--- a/docs/_data/table/cn/user-configuration.yml
+++ b/docs/_data/table/cn/user-configuration.yml
@@ -28,8 +28,8 @@ alluxio.user.file.worker.client.threads:
文件worker client从worker读取数据时使用的线程数目。
alluxio.user.file.write.location.policy.class:
选择worker进行写文件数据块时的默认定位机制。
-alluxio.user.file.write.capacity.reserved.ratio:
- 当用户选择LocalFirstWithoutEvictionPolicy作为写文件数据块的定位机制时，使用文件块大小和这个参数的乘机来作为worker的预留存储空间。值为[0,1]。
+alluxio.user.file.write.capacity.reserved:
+ 当用户选择LocalFirstAvoidEvictionPolicy作为写文件数据块的定位机制时,用户需要配置worker预留一些数据量来保证数据的存储，默认是0MB。
alluxio.user.file.readtype.default:
创建Alluxio文件时的默认读类型。可选值为`CACHE_PROMOTE` (如果数据已经在Alluxio存储内，将其移动到最高存储层，如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层)、`CACHE` (如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层), `NO_CACHE` (数据不与Alluxio交互，如果是从Alluxio中进行读取，将不会发生数据块迁移或者剔除)。
alluxio.user.file.writetype.default:
diff --git a/docs/_data/table/en/user-configuration.yml b/docs/_data/table/en/user-configuration.yml
index dfc06fcd448c..2acf9b3b955d 100644
--- a/docs/_data/table/en/user-configuration.yml
+++ b/docs/_data/table/en/user-configuration.yml
@@ -30,9 +30,9 @@ alluxio.user.file.worker.client.threads:
How many threads to use for file worker clients to read from workers.
alluxio.user.file.write.location.policy.class:
The default location policy for choosing workers for writing a file's blocks
-alluxio.user.file.write.capacity.reserved.ratio:
- The portion of space reserced in worker when user use the LocalFirstWithoutEvictionPolicy class
- as file write location policy (a value between 0 and 1).
+alluxio.user.file.write.capacity.reserved:
+ The portion of space reserced in worker when user use the LocalFirstAvoidEvictionPolicy class
+ as file write location policy, default 0 MB.
alluxio.user.file.readtype.default:
Default read type when creating Alluxio files.
Valid options are `CACHE_PROMOTE` (move data to highest tier if already in Alluxio storage,
diff --git a/docs/_data/table/user-configuration.csv b/docs/_data/table/user-configuration.csv
index 8a9e44c271db..39588a4156e7 100644
--- a/docs/_data/table/user-configuration.csv
+++ b/docs/_data/table/user-configuration.csv
@@ -12,7 +12,7 @@ alluxio.user.file.master.client.threads,10
alluxio.user.file.waitcompleted.poll.ms,1000
alluxio.user.file.worker.client.threads,10
alluxio.user.file.write.location.policy.class,alluxio.client.file.policy.LocalFirstPolicy
-alluxio.user.file.write.capacity.reserved.ratio,1
+alluxio.user.file.write.capacity.reserved,0MB
alluxio.user.file.readtype.default,CACHE_PROMOTE
alluxio.user.file.writetype.default,MUST_CACHE
alluxio.user.heartbeat.interval.ms,1000
From 9a16b089ab940384651398db5bb27eac21dfd031 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 13:59:39 +0800
Subject: [PATCH 07/17] Fixed Checkstyle errors
---
core/common/src/main/java/alluxio/PropertyKey.java | 3 ++-
docs/_data/table/cn/user-configuration.yml | 2 +-
docs/_data/table/en/user-configuration.yml | 2 +-
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/core/common/src/main/java/alluxio/PropertyKey.java b/core/common/src/main/java/alluxio/PropertyKey.java
index 869e3ec18f1e..b2c570c42353 100644
--- a/core/common/src/main/java/alluxio/PropertyKey.java
+++ b/core/common/src/main/java/alluxio/PropertyKey.java
@@ -252,7 +252,8 @@ public enum PropertyKey {
Name.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS, 300 * Constants.SECOND_MS),
USER_FILE_WRITE_LOCATION_POLICY(Name.USER_FILE_WRITE_LOCATION_POLICY,
"alluxio.client.file.policy.LocalFirstPolicy"),
- USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES(Name.USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES, "0MB"),
+ USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES(
+ Name.USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES, "0MB"),
USER_FILE_WRITE_TYPE_DEFAULT(Name.USER_FILE_WRITE_TYPE_DEFAULT, "MUST_CACHE"),
USER_HEARTBEAT_INTERVAL_MS(Name.USER_HEARTBEAT_INTERVAL_MS, 1000),
USER_LINEAGE_ENABLED(Name.USER_LINEAGE_ENABLED, false),
diff --git a/docs/_data/table/cn/user-configuration.yml b/docs/_data/table/cn/user-configuration.yml
index 2741f46e63ca..621f05073164 100644
--- a/docs/_data/table/cn/user-configuration.yml
+++ b/docs/_data/table/cn/user-configuration.yml
@@ -28,7 +28,7 @@ alluxio.user.file.worker.client.threads:
文件worker client从worker读取数据时使用的线程数目。
alluxio.user.file.write.location.policy.class:
选择worker进行写文件数据块时的默认定位机制。
-alluxio.user.file.write.capacity.reserved:
+alluxio.user.file.write.capacity.reserved.size.bytes:
当用户选择LocalFirstAvoidEvictionPolicy作为写文件数据块的定位机制时,用户需要配置worker预留一些数据量来保证数据的存储，默认是0MB。
alluxio.user.file.readtype.default:
创建Alluxio文件时的默认读类型。可选值为`CACHE_PROMOTE` (如果数据已经在Alluxio存储内，将其移动到最高存储层，如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层)、`CACHE` (如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层), `NO_CACHE` (数据不与Alluxio交互，如果是从Alluxio中进行读取，将不会发生数据块迁移或者剔除)。
diff --git a/docs/_data/table/en/user-configuration.yml b/docs/_data/table/en/user-configuration.yml
index 2acf9b3b955d..6592edff6f2b 100644
--- a/docs/_data/table/en/user-configuration.yml
+++ b/docs/_data/table/en/user-configuration.yml
@@ -30,7 +30,7 @@ alluxio.user.file.worker.client.threads:
How many threads to use for file worker clients to read from workers.
alluxio.user.file.write.location.policy.class:
The default location policy for choosing workers for writing a file's blocks
-alluxio.user.file.write.capacity.reserved:
+alluxio.user.file.write.capacity.reserved.size.bytes:
The portion of space reserced in worker when user use the LocalFirstAvoidEvictionPolicy class
as file write location policy, default 0 MB.
alluxio.user.file.readtype.default:
From 065e2b11e8c0b05dd87bd4628d34d98818fd3355 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 14:04:32 +0800
Subject: [PATCH 08/17] Update user-configuration
---
docs/_data/table/user-configuration.csv | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/_data/table/user-configuration.csv b/docs/_data/table/user-configuration.csv
index 39588a4156e7..e2ed1efb6c4c 100644
--- a/docs/_data/table/user-configuration.csv
+++ b/docs/_data/table/user-configuration.csv
@@ -12,7 +12,7 @@ alluxio.user.file.master.client.threads,10
alluxio.user.file.waitcompleted.poll.ms,1000
alluxio.user.file.worker.client.threads,10
alluxio.user.file.write.location.policy.class,alluxio.client.file.policy.LocalFirstPolicy
-alluxio.user.file.write.capacity.reserved,0MB
+alluxio.user.file.write.capacity.reserved,size.bytes,0MB
alluxio.user.file.readtype.default,CACHE_PROMOTE
alluxio.user.file.writetype.default,MUST_CACHE
alluxio.user.heartbeat.interval.ms,1000
From b8825672784246711b51a704d24fc8189975fbce Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 15:58:29 +0800
Subject: [PATCH 09/17] Fixed a word
---
docs/_data/table/en/user-configuration.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/_data/table/en/user-configuration.yml b/docs/_data/table/en/user-configuration.yml
index 6592edff6f2b..7bf40112820f 100644
--- a/docs/_data/table/en/user-configuration.yml
+++ b/docs/_data/table/en/user-configuration.yml
@@ -31,7 +31,7 @@ alluxio.user.file.worker.client.threads:
alluxio.user.file.write.location.policy.class:
The default location policy for choosing workers for writing a file's blocks
alluxio.user.file.write.capacity.reserved.size.bytes:
- The portion of space reserced in worker when user use the LocalFirstAvoidEvictionPolicy class
+ The portion of space reserved in worker when user use the LocalFirstAvoidEvictionPolicy class
as file write location policy, default 0 MB.
alluxio.user.file.readtype.default:
Default read type when creating Alluxio files.
From d4dc9e1d697c16fe4d39b79b14ffd280b257e147 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 16:48:28 +0800
Subject: [PATCH 10/17] Remove the method getAvailableCapacity in
BlockWorkerInfo
---
.../java/alluxio/client/block/BlockWorkerInfo.java | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java b/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
index 0a4d08481869..ef4a249839c6 100644
--- a/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
+++ b/core/client/src/main/java/alluxio/client/block/BlockWorkerInfo.java
@@ -11,8 +11,6 @@
package alluxio.client.block;
-import alluxio.Configuration;
-import alluxio.PropertyKey;
import alluxio.annotation.PublicApi;
import alluxio.wire.WorkerNetAddress;
@@ -29,7 +27,6 @@ public final class BlockWorkerInfo {
private final WorkerNetAddress mNetAddress;
private final long mCapacityBytes;
private final long mUsedBytes;
- private long mUserFileWriteCapacityReserved;
/**
* Constructs the block worker information.
@@ -58,15 +55,6 @@ public long getCapacityBytes() {
return mCapacityBytes;
}
- /**
- * @return the available bytes of the worker
- */
- public long getAvailableBytes() {
- mUserFileWriteCapacityReserved = Configuration
- .getBytes(PropertyKey.USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES);
- return mCapacityBytes - mUsedBytes - mUserFileWriteCapacityReserved;
- }
-
/**
* @return the used bytes of the worker
*/
From 3e50fdb41440dc7afe8851720886dab87bb30c16 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 16:50:07 +0800
Subject: [PATCH 11/17] update the user write file local first avoid eviction
policy reserved configuration key
---
core/common/src/main/java/alluxio/PropertyKey.java | 8 ++++----
docs/_data/table/cn/user-configuration.yml | 2 +-
docs/_data/table/en/user-configuration.yml | 2 +-
docs/_data/table/user-configuration.csv | 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/core/common/src/main/java/alluxio/PropertyKey.java b/core/common/src/main/java/alluxio/PropertyKey.java
index b2c570c42353..8bbfc8b7ce0a 100644
--- a/core/common/src/main/java/alluxio/PropertyKey.java
+++ b/core/common/src/main/java/alluxio/PropertyKey.java
@@ -252,8 +252,8 @@ public enum PropertyKey {
Name.USER_FILE_WORKER_CLIENT_POOL_GC_THRESHOLD_MS, 300 * Constants.SECOND_MS),
USER_FILE_WRITE_LOCATION_POLICY(Name.USER_FILE_WRITE_LOCATION_POLICY,
"alluxio.client.file.policy.LocalFirstPolicy"),
- USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES(
- Name.USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES, "0MB"),
+ USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES(
+ Name.USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES, "0MB"),
USER_FILE_WRITE_TYPE_DEFAULT(Name.USER_FILE_WRITE_TYPE_DEFAULT, "MUST_CACHE"),
USER_HEARTBEAT_INTERVAL_MS(Name.USER_HEARTBEAT_INTERVAL_MS, 1000),
USER_LINEAGE_ENABLED(Name.USER_LINEAGE_ENABLED, false),
@@ -663,8 +663,8 @@ public static final class Name {
"alluxio.user.file.worker.client.pool.gc.threshold.ms";
public static final String USER_FILE_WRITE_LOCATION_POLICY =
"alluxio.user.file.write.location.policy.class";
- public static final String USER_FILE_WRITE_CAPACITY_RESERVED_SIZE_BYTES =
- "alluxio.user.file.write.capacity.reserved.size.bytes";
+ public static final String USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES =
+ "alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes";
public static final String USER_FILE_WRITE_TYPE_DEFAULT = "alluxio.user.file.writetype.default";
public static final String USER_HEARTBEAT_INTERVAL_MS = "alluxio.user.heartbeat.interval.ms";
public static final String USER_LINEAGE_ENABLED = "alluxio.user.lineage.enabled";
diff --git a/docs/_data/table/cn/user-configuration.yml b/docs/_data/table/cn/user-configuration.yml
index 621f05073164..73af839f0cf4 100644
--- a/docs/_data/table/cn/user-configuration.yml
+++ b/docs/_data/table/cn/user-configuration.yml
@@ -28,7 +28,7 @@ alluxio.user.file.worker.client.threads:
文件worker client从worker读取数据时使用的线程数目。
alluxio.user.file.write.location.policy.class:
选择worker进行写文件数据块时的默认定位机制。
-alluxio.user.file.write.capacity.reserved.size.bytes:
+alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes:
当用户选择LocalFirstAvoidEvictionPolicy作为写文件数据块的定位机制时,用户需要配置worker预留一些数据量来保证数据的存储，默认是0MB。
alluxio.user.file.readtype.default:
创建Alluxio文件时的默认读类型。可选值为`CACHE_PROMOTE` (如果数据已经在Alluxio存储内，将其移动到最高存储层，如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层)、`CACHE` (如果数据需要从底层存储进行读取，将其写到本地Alluxio的最高存储层), `NO_CACHE` (数据不与Alluxio交互，如果是从Alluxio中进行读取，将不会发生数据块迁移或者剔除)。
diff --git a/docs/_data/table/en/user-configuration.yml b/docs/_data/table/en/user-configuration.yml
index 7bf40112820f..cc4070c6f003 100644
--- a/docs/_data/table/en/user-configuration.yml
+++ b/docs/_data/table/en/user-configuration.yml
@@ -30,7 +30,7 @@ alluxio.user.file.worker.client.threads:
How many threads to use for file worker clients to read from workers.
alluxio.user.file.write.location.policy.class:
The default location policy for choosing workers for writing a file's blocks
-alluxio.user.file.write.capacity.reserved.size.bytes:
+alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes:
The portion of space reserved in worker when user use the LocalFirstAvoidEvictionPolicy class
as file write location policy, default 0 MB.
alluxio.user.file.readtype.default:
diff --git a/docs/_data/table/user-configuration.csv b/docs/_data/table/user-configuration.csv
index e2ed1efb6c4c..afb4437ed1bf 100644
--- a/docs/_data/table/user-configuration.csv
+++ b/docs/_data/table/user-configuration.csv
@@ -12,7 +12,7 @@ alluxio.user.file.master.client.threads,10
alluxio.user.file.waitcompleted.poll.ms,1000
alluxio.user.file.worker.client.threads,10
alluxio.user.file.write.location.policy.class,alluxio.client.file.policy.LocalFirstPolicy
-alluxio.user.file.write.capacity.reserved,size.bytes,0MB
+alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes,0MB
alluxio.user.file.readtype.default,CACHE_PROMOTE
alluxio.user.file.writetype.default,MUST_CACHE
alluxio.user.heartbeat.interval.ms,1000
From 6bddccf0565c86600e5df6d6d8992365a06b2ed9 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 16:50:50 +0800
Subject: [PATCH 12/17] add the method getAvailableBytes
---
.../policy/LocalFirstAvoidEvictionPolicy.java | 18 ++++++++++++++++--
1 file changed, 16 insertions(+), 2 deletions(-)
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index 6baaced4d10b..38da7f36aed6 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -11,6 +11,8 @@
package alluxio.client.file.policy;
+import alluxio.Configuration;
+import alluxio.PropertyKey;
import alluxio.client.block.BlockWorkerInfo;
import alluxio.util.network.NetworkAddressUtils;
import alluxio.wire.WorkerNetAddress;
@@ -45,7 +47,7 @@ public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerIn
for (BlockWorkerInfo workerInfo : workerInfoList) {
if (workerInfo.getNetAddress().getHost().equals(mLocalHostName)) {
localWorkerNetAddress = workerInfo.getNetAddress();
- if (workerInfo.getAvailableBytes() >= blockSizeBytes) {
+ if (getAvailableBytes(workerInfo) >= blockSizeBytes) {
return localWorkerNetAddress;
}
}
@@ -55,13 +57,25 @@ public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerIn
List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList);
Collections.shuffle(shuffledWorkers);
for (BlockWorkerInfo workerInfo : shuffledWorkers) {
- if (workerInfo.getAvailableBytes() >= blockSizeBytes) {
+ if (getAvailableBytes(workerInfo) >= blockSizeBytes) {
return workerInfo.getNetAddress();
}
}
return localWorkerNetAddress;
}
+ /**
+ * @param workerInfo BlockWorkerInfo of the worker
+ * @return the available bytes of the worker
+ */
+ private long getAvailableBytes(BlockWorkerInfo workerInfo) {
+ long mUserFileWriteCapacityReserved = Configuration
+ .getBytes(PropertyKey.USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES);
+ long mCapacityBytes = workerInfo.getCapacityBytes();
+ long mUsedBytes = workerInfo.getUsedBytes();
+ return mCapacityBytes - mUsedBytes - mUserFileWriteCapacityReserved;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
From 0d46e58b1df623eadc5b8115c2aa7e6033d72fa4 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Mon, 19 Dec 2016 17:07:14 +0800
Subject: [PATCH 13/17] add the specify of
USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES
---
.../client/file/policy/LocalFirstAvoidEvictionPolicy.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index 38da7f36aed6..0f28f0468194 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -28,6 +28,8 @@
* A policy that returns local host first, and if the local worker doesn't have enough availability,
* it randomly picks a worker from the active workers list for each block write.
* If No worker meets the demands, return local host.
+ * USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES is use to reserved some space of the worker
+ * to store the block, for the values mCapacityBytes minus mUsedBytes is not the available bytes.
*/
@ThreadSafe
public class LocalFirstAvoidEvictionPolicy implements FileWriteLocationPolicy {
From c80466e31e6b9b16081d45cb6b471911627980f2 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Tue, 20 Dec 2016 09:53:10 +0800
Subject: [PATCH 14/17] Add modifier final to public class
---
.../client/file/policy/LocalFirstAvoidEvictionPolicy.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index 0f28f0468194..7d3856183a0c 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -28,11 +28,11 @@
* A policy that returns local host first, and if the local worker doesn't have enough availability,
* it randomly picks a worker from the active workers list for each block write.
* If No worker meets the demands, return local host.
- * USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES is use to reserved some space of the worker
+ * USER_FILE_WRITE_AVOID_EVICTION_POLICY_RESERVED_BYTES is used to reserve some space of the worker
* to store the block, for the values mCapacityBytes minus mUsedBytes is not the available bytes.
*/
@ThreadSafe
-public class LocalFirstAvoidEvictionPolicy implements FileWriteLocationPolicy {
+public final class LocalFirstAvoidEvictionPolicy implements FileWriteLocationPolicy {
private String mLocalHostName;
/**
* Constructs a {@link LocalFirstAvoidEvictionPolicy}.
From 716452f30711619bc1055fd2c7d3fbbcd7590705 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Thu, 22 Dec 2016 10:51:40 +0800
Subject: [PATCH 15/17] Add describe info of getAvailableBytes
---
.../client/file/policy/LocalFirstAvoidEvictionPolicy.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index 7d3856183a0c..6b940217c4bf 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -67,6 +67,10 @@ public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerIn
}
/**
+ * The information of BlockWorkerInfo is update after a file complete write. To avoid evict,
+ * user should configure "alluxio.user.file.write.avoid.eviction.policy.reserved.size.bytes"
+ * to reserve some space to store the block.
+ *
* @param workerInfo BlockWorkerInfo of the worker
* @return the available bytes of the worker
*/
From a263a4e56f309c749c5382b9a9a78bf976d626fe Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Thu, 29 Dec 2016 11:25:08 +0800
Subject: [PATCH 16/17] If no worker on the local should return a random worker
---
.../client/file/policy/LocalFirstAvoidEvictionPolicy.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index 6b940217c4bf..603177bd7b6a 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -63,7 +63,10 @@ public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerIn
return workerInfo.getNetAddress();
}
}
- return localWorkerNetAddress;
+ if (localWorkerNetAddress == null && shuffledWorkers.size() > 0) {
+ return shuffledWorkers.get(0).getNetAddress();
+ }
+ return null;
}
/**
From 6699389597d06621f209806f2d4b52fec2088943 Mon Sep 17 00:00:00 2001
From: gjhkael <guojhltt@gmail.com>
Date: Thu, 29 Dec 2016 15:22:11 +0800
Subject: [PATCH 17/17] Fix test bug
---
.../client/file/policy/LocalFirstAvoidEvictionPolicy.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
index 603177bd7b6a..1b80c791b991 100644
--- a/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
+++ b/core/client/src/main/java/alluxio/client/file/policy/LocalFirstAvoidEvictionPolicy.java
@@ -66,7 +66,7 @@ public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerIn
if (localWorkerNetAddress == null && shuffledWorkers.size() > 0) {
return shuffledWorkers.get(0).getNetAddress();
}
- return null;
+ return localWorkerNetAddress;
}
/**