Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

direct-io test in block-cache data validation suite #1490

Merged
merged 13 commits into from
Aug 8, 2024
Merged

direct-io test in block-cache data validation suite #1490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
40357e3
adding direct_io data validation test with block cache
vibhansa-msft Aug 5, 2024
b39cf33
enable git clone test
vibhansa-msft Aug 5, 2024
5af80a8
Merge remote-tracking branch 'origin/main' into vibhansa/block_cache_…
vibhansa-msft Aug 5, 2024
0075f48
Add fuse2 for block-cache data validation stage
vibhansa-msft Aug 5, 2024
acd9ed1
simplify direct io mount tests
vibhansa-msft Aug 5, 2024
f5eea5f
Correcting direct_io condition
vibhansa-msft Aug 5, 2024
2b7d62b
Correcting direct_io condition
vibhansa-msft Aug 5, 2024
2e6540e
Correcting direct_io condition
vibhansa-msft Aug 5, 2024
b36b541
Correcting direct_io condition
vibhansa-msft Aug 5, 2024
5e0b8f4
Correcting direct_io condition
vibhansa-msft Aug 5, 2024
31580e6
Correction
vibhansa-msft Aug 5, 2024
735bbe2
Disable git clone again
vibhansa-msft Aug 5, 2024
0c631bf
Correcting stage name
vibhansa-msft Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions azure-pipeline-templates/e2e-tests-block-cache.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ parameters:
- name: quick_test
type: boolean
default: true
- name: mnt_flags
type: string
default: ""
- name: verbose_log
type: boolean
default: false
Expand All @@ -48,6 +51,7 @@ steps:
cat ${{ parameters.config_file }}
displayName: 'Print config file'

# run below step only if direct_io is false
- template: 'mount.yml'
parameters:
working_dir: $(WORK_DIR)
Expand All @@ -56,7 +60,7 @@ steps:
prefix: ${{ parameters.idstring }}
mountStep:
script: |
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 ${{ parameters.mnt_flags }}

- script: |
for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${{ parameters.mount_dir }}/myfile_{}'
Expand Down Expand Up @@ -96,7 +100,7 @@ steps:
ro_mount: true
mountStep:
script: |
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro ${{ parameters.mnt_flags }}

- script: |
md5sum ${{ parameters.mount_dir }}/myfile_* > $(WORK_DIR)/md5sum_block_cache.txt
Expand Down Expand Up @@ -127,7 +131,7 @@ steps:
ro_mount: true
mountStep:
script: |
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR)
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) ${{ parameters.mnt_flags }}

- script: |
for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'cp ${{ parameters.mount_dir }}/myfile_{} ${{ parameters.mount_dir }}/myfileCopy_{}'
Expand Down Expand Up @@ -166,7 +170,7 @@ steps:
ro_mount: true
mountStep:
script: |
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR)
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) ${{ parameters.mnt_flags }}

- script: |
rm -rf $(WORK_DIR)/localfile*
Expand Down Expand Up @@ -218,7 +222,7 @@ steps:
ro_mount: true
mountStep:
script: |
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR)
$(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) ${{ parameters.mnt_flags }}

- script: |
md5sum $(WORK_DIR)/localfile* > $(WORK_DIR)/md5sum_local_modified.txt
Expand Down
2 changes: 1 addition & 1 deletion azure-pipeline-templates/mount.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ parameters:
default: 'Test'
- name: ro_mount
type: boolean
default: false
default: false

steps:

Expand Down
85 changes: 85 additions & 0 deletions blobfuse2-nightly.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1349,6 +1349,89 @@ stages:
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
fuselib: 'libfuse3-dev'
tags: 'fuse3'
Ubuntu-20:
AgentName: 'blobfuse-ubuntu20'
containerName: 'test-cnt-ubn-20'
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
fuselib: 'libfuse-dev'
tags: 'fuse2'

pool:
name: "blobfuse-ubuntu-pool"
demands:
- ImageOverride -equals $(AgentName)

variables:
- group: NightlyBlobFuse
- name: ROOT_DIR
value: "/usr/pipeline/workv2"
- name: WORK_DIR
value: "/usr/pipeline/workv2/go/src/azure-storage-fuse"
- name: skipComponentGovernanceDetection
value: true
- name: MOUNT_DIR
value: "/usr/pipeline/workv2/blob_mnt"
- name: TEMP_DIR
value: "/usr/pipeline/workv2/temp"
- name: BLOBFUSE2_CFG
value: "/usr/pipeline/workv2/blobfuse2.yaml"
- name: GOPATH
value: "/usr/pipeline/workv2/go"

steps:
- template: 'azure-pipeline-templates/setup.yml'
parameters:
tags: $(tags)
installStep:
script: |
sudo apt-get update --fix-missing
sudo apt update
sudo apt-get install cmake gcc $(fuselib) git parallel -y
if [ $(tags) == "fuse2" ]; then
sudo apt-get install fuse -y
else
sudo apt-get install fuse3 -y
fi
displayName: 'Install fuse'

- template: 'azure-pipeline-templates/e2e-tests-block-cache.yml'
parameters:
conf_template: azure_key.yaml
config_file: $(BLOBFUSE2_CFG)
container: $(containerName)
idstring: Block_Blob
adls: false
account_name: $(NIGHTLY_STO_BLOB_ACC_NAME)
account_key: $(NIGHTLY_STO_BLOB_ACC_KEY)
account_type: block
account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net
distro_name: $(AgentName)
quick_test: false
verbose_log: ${{ parameters.verbose_log }}
clone: true
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
temp_dir: $(TEMP_DIR)
mount_dir: $(MOUNT_DIR)

- stage: BlockCacheDataDirectIOValidation
jobs:
# Ubuntu Tests
- job: Set_1
timeoutInMinutes: 300
strategy:
matrix:
Ubuntu-20:
AgentName: 'blobfuse-ubuntu20'
containerName: 'test-cnt-ubn-20'
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20)
fuselib: 'libfuse-dev'
tags: 'fuse2'
Ubuntu-22:
AgentName: 'blobfuse-ubuntu22'
containerName: 'test-cnt-ubn-22'
adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22)
fuselib: 'libfuse3-dev'
tags: 'fuse3'

pool:
name: "blobfuse-ubuntu-pool"
Expand Down Expand Up @@ -1406,6 +1489,8 @@ stages:
# TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable
temp_dir: $(TEMP_DIR)
mount_dir: $(MOUNT_DIR)
mnt_flags: "-o direct_io"


- stage: FNSDataValidation
jobs:
Expand Down
Loading