From f58b918fc1b042a3f5316977d307a2ea97ce0a33 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sat, 9 Jul 2022 18:34:18 -0300 Subject: [PATCH 01/16] file: refactored to move the native bits away Signed-off-by: Leonardo Alminana --- include/chunkio/cio_file.h | 1 + src/cio_file.c | 491 ++++++++----------------------------- 2 files changed, 110 insertions(+), 382 deletions(-) diff --git a/include/chunkio/cio_file.h b/include/chunkio/cio_file.h index cf9f8bc..a33040b 100644 --- a/include/chunkio/cio_file.h +++ b/include/chunkio/cio_file.h @@ -77,5 +77,6 @@ int cio_file_up(struct cio_chunk *ch); int cio_file_up_force(struct cio_chunk *ch); int cio_file_lookup_user(char *user, void **result); int cio_file_lookup_group(char *group, void **result); +int cio_file_update_size(struct cio_file *cf); #endif diff --git a/src/cio_file.c b/src/cio_file.c index 776716c..270dc29 100644 --- a/src/cio_file.c +++ b/src/cio_file.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -271,8 +272,8 @@ static int munmap_file(struct cio_ctx *ctx, struct cio_chunk *ch) } /* Unmap file */ - munmap(cf->map, cf->alloc_size); - cf->map = NULL; + cio_file_native_unmap(cf); + cf->data_size = 0; cf->alloc_size = 0; @@ -288,14 +289,13 @@ static int munmap_file(struct cio_ctx *ctx, struct cio_chunk *ch) */ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) { - int ret; - int oflags = 0; - size_t fs_size = 0; - ssize_t content_size; - struct stat fst; + ssize_t content_size; + size_t fs_size; + int ret; struct cio_file *cf; cf = (struct cio_file *) ch->backend; + if (cf->map != NULL) { return CIO_OK; } @@ -306,26 +306,20 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) * fstat(2) of the file descriptor. */ + fs_size = 0; + if (size > 0) { fs_size = size; } else { - ret = fstat(cf->fd, &fst); - if (ret == -1) { + /* Get file size from the file system */ + ret = cio_file_native_get_size(cf, &fs_size); + + if (ret != CIO_OK) { cio_errno(); + return CIO_ERROR; } - - /* Get file size from the file system */ - fs_size = fst.st_size; - } - - /* Mmap */ - if (cf->flags & CIO_OPEN_RW) { - oflags = PROT_READ | PROT_WRITE; - } - else if (cf->flags & CIO_OPEN_RD) { - oflags = PROT_READ; } /* If the file is not empty, use file size for the memory map */ @@ -363,26 +357,29 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) cf->alloc_size = size; /* Map the file */ - cf->map = mmap(0, size, oflags, MAP_SHARED, cf->fd, 0); - if (cf->map == MAP_FAILED) { - cio_errno(); - cf->map = NULL; + ret = cio_file_native_map(cf, cf->alloc_size); + + if (ret != CIO_OK) { cio_log_error(ctx, "cannot mmap/read chunk '%s'", cf->path); + return CIO_ERROR; } /* check content data size */ if (fs_size > 0) { content_size = cio_file_st_get_content_size(cf->map, fs_size); + if (content_size == -1) { cio_log_error(ctx, "invalid content size %s", cf->path); - munmap(cf->map, cf->alloc_size); - cf->map = NULL; + + cio_file_native_unmap(cf); + cf->data_size = 0; cf->alloc_size = 0; return CIO_CORRUPTED; } + cf->data_size = content_size; cf->fs_size = fs_size; } @@ -392,13 +389,16 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) } ret = cio_file_format_check(ch, cf, cf->flags); + if (ret != 0) { cio_log_error(ctx, "format check failed: %s/%s", ch->st->name, ch->name); - munmap(cf->map, cf->alloc_size); - cf->map = NULL; + + cio_file_native_unmap(cf); + cf->data_size = 0; cf->alloc_size = 0; + return CIO_CORRUPTED; } @@ -413,250 +413,17 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) int cio_file_lookup_user(char *user, void **result) { - long query_buffer_size; - struct passwd *query_result; - char *query_buffer; - struct passwd passwd_entry; - int api_result; - - if (user == NULL) { - *result = calloc(1, sizeof(uid_t)); - - if (*result == NULL) { - cio_errno(); - - return CIO_ERROR; - } - - **(uid_t **) result = (uid_t) -1; - } - - query_buffer_size = sysconf(_SC_GETPW_R_SIZE_MAX); - - if (query_buffer_size == -1) { - query_buffer_size = 4096 * 10; - } - - query_buffer = calloc(1, query_buffer_size); - - if (query_buffer == NULL) { - return CIO_ERROR; - } - - query_result = NULL; - - api_result = getpwnam_r(user, &passwd_entry, query_buffer, - query_buffer_size, &query_result); - - if (api_result != 0 || query_result == NULL) { - cio_errno(); - - free(query_buffer); - - return CIO_ERROR; - } - - *result = calloc(1, sizeof(uid_t)); - - if (*result == NULL) { - cio_errno(); - - free(query_buffer); - - return CIO_ERROR; - } - - **(uid_t **) result = query_result->pw_uid; - - free(query_buffer); - - return CIO_OK; + return cio_file_native_lookup_user(user, result); } int cio_file_lookup_group(char *group, void **result) { - long query_buffer_size; - struct group *query_result; - char *query_buffer; - struct group group_entry; - int api_result; - - if (group == NULL) { - *result = calloc(1, sizeof(gid_t)); - - if (*result == NULL) { - cio_errno(); - - return CIO_ERROR; - } - - **(gid_t **) result = (gid_t) -1; - } - - query_buffer_size = sysconf(_SC_GETGR_R_SIZE_MAX); - - if (query_buffer_size == -1) { - query_buffer_size = 4096 * 10; - } - - query_buffer = calloc(1, query_buffer_size); - - if (query_buffer == NULL) { - return CIO_ERROR; - } - - query_result = NULL; - - api_result = getgrnam_r(group, &group_entry, query_buffer, - query_buffer_size, &query_result); - - if (api_result != 0 || query_result == NULL) { - cio_errno(); - - free(query_buffer); - - return CIO_ERROR; - } - - *result = calloc(1, sizeof(gid_t)); - - if (*result == NULL) { - cio_errno(); - - free(query_buffer); - - return CIO_ERROR; - } - - **(gid_t **) result = query_result->gr_gid; - - free(query_buffer); - - return CIO_OK; -} - -static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path) -{ - mode_t filesystem_acl; - gid_t numeric_group; - uid_t numeric_user; - char *connector; - int result; - char *group; - char *user; - - numeric_group = -1; - numeric_user = -1; - - if (ctx->processed_user != NULL) { - numeric_user = *(uid_t *) ctx->processed_user; - } - - if (ctx->processed_group != NULL) { - numeric_group = *(gid_t *) ctx->processed_group; - } - - if (numeric_user != -1 || numeric_group != -1) { - result = chown(path, numeric_user, numeric_group); - - if (result == -1) { - cio_errno(); - - user = ctx->options.user; - group = ctx->options.group; - connector = "with group"; - - if (user == NULL) { - user = ""; - connector = ""; - } - - if (group == NULL) { - group = ""; - connector = ""; - } - - cio_log_error(ctx, "cannot change ownership of %s to %s %s %s", - path, user, connector, group); - - return CIO_ERROR; - } - } - - if (ctx->options.chmod != NULL) { - filesystem_acl = strtoul(ctx->options.chmod, NULL, 8); - - result = chmod(path, filesystem_acl); - - if (result == -1) { - cio_errno(); - cio_log_error(ctx, "cannot change acl of %s to %s", - path, ctx->options.user); - - return CIO_ERROR; - } - } - - return CIO_OK; -} - -/* Open file system file, set file descriptor and file size */ -static int file_open(struct cio_ctx *ctx, struct cio_file *cf) -{ - int ret; - struct stat st; - - if (cf->map || cf->fd > 0) { - return -1; - } - - /* Open file descriptor */ - if (cf->flags & CIO_OPEN_RW) { - cf->fd = open(cf->path, O_RDWR | O_CREAT, (mode_t) 0600); - } - else if (cf->flags & CIO_OPEN_RD) { - cf->fd = open(cf->path, O_RDONLY); - } - - if (cf->fd == -1) { - cio_errno(); - cio_log_error(ctx, "cannot open/create %s", cf->path); - return -1; - } - - ret = apply_file_ownership_and_acl_settings(ctx, cf->path); - if (ret == CIO_ERROR) { - cio_errno(); - close(cf->fd); - cf->fd = -1; - return -1; - } - - /* Store the current real size */ - ret = fstat(cf->fd, &st); - if (ret == -1) { - cio_errno(); - close(cf->fd); - cf->fd = -1; - return -1; - } - cf->fs_size = st.st_size; - - return 0; + return cio_file_native_lookup_group(group, result); } int cio_file_read_prepare(struct cio_ctx *ctx, struct cio_chunk *ch) - { - int ret; - struct cio_file *cf = ch->backend; - - if (!cf->map) { - ret = mmap_file(ctx, ch, 0); - return ret; - } - - return 0; + return mmap_file(ctx, ch, 0); } int cio_file_content_copy(struct cio_chunk *ch, @@ -727,17 +494,16 @@ static inline int open_and_up(struct cio_ctx *ctx) */ size_t cio_file_real_size(struct cio_file *cf) { - int ret; - struct stat st; + size_t file_size; + int ret; - /* Store the current real size */ - ret = stat(cf->path, &st); - if (ret == -1) { - cio_errno(); + ret = cio_file_native_get_size(cf, &file_size); + + if (ret != CIO_OK) { return 0; } - return st.st_size; + return file_size; } /* @@ -824,7 +590,7 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, } /* Open file (file descriptor and set file size) */ - ret = file_open(ctx, cf); + ret = cio_file_native_open(ctx, cf); if (ret == -1) { cio_file_close(ch, CIO_FALSE); *err = CIO_ERROR; @@ -876,7 +642,7 @@ static int _cio_file_up(struct cio_chunk *ch, int enforced) } /* Open file */ - ret = file_open(ch->ctx, cf); + ret = cio_file_native_open(ch->ctx, cf); if (ret == -1) { cio_log_error(ch->ctx, "[cio file] cannot open chunk: %s/%s", ch->st->name, ch->name); @@ -904,8 +670,7 @@ static int _cio_file_up(struct cio_chunk *ch, int enforced) * descriptor, we never delete the Chunk at this stage since * the caller must take that action. */ - close(cf->fd); - cf->fd = -1; + cio_file_native_close(cf); } return ret; @@ -932,14 +697,28 @@ int cio_file_up_force(struct cio_chunk *ch) return _cio_file_up(ch, CIO_FALSE); } +int cio_file_update_size(struct cio_file *cf) +{ + int result; + + result = cio_file_native_get_size(cf, &cf->fs_size); + + if (result != CIO_OK) { + cf->fs_size = 0; + } + + return result; +} + /* Release memory and file descriptor resources but keep context */ int cio_file_down(struct cio_chunk *ch) { - int ret; - struct stat st; - struct cio_file *cf = (struct cio_file *) ch->backend; + int ret; + struct cio_file *cf; - if (!cf->map) { + cf = (struct cio_file *) ch->backend; + + if (cf->map == NULL) { cio_log_error(ch->ctx, "[cio file] file is not mapped: %s/%s", ch->st->name, ch->name); return -1; @@ -951,30 +730,27 @@ int cio_file_down(struct cio_chunk *ch) /* Allocated map size is zero */ cf->alloc_size = 0; - /* Get file size */ - ret = fstat(cf->fd, &st); - if (ret == -1) { + /* Update the file size */ + ret = cio_file_update_size(cf); + + if (ret != CIO_OK) { cio_errno(); - cf->fs_size = 0; - } - else { - cf->fs_size = st.st_size; } /* Close file descriptor */ - close(cf->fd); - cf->fd = -1; - cf->map = NULL; + cio_file_native_close(cf); return 0; } void cio_file_close(struct cio_chunk *ch, int delete) { - int ret; - struct cio_file *cf = (struct cio_file *) ch->backend; + int ret; + struct cio_file *cf; - if (!cf) { + cf = (struct cio_file *) ch->backend; + + if (cf == NULL) { return; } @@ -983,7 +759,8 @@ void cio_file_close(struct cio_chunk *ch, int delete) /* Should we delete the content from the file system ? */ if (delete == CIO_TRUE) { - ret = unlink(cf->path); + ret = cio_file_native_delete(cf); + if (ret == -1) { cio_errno(); cio_log_error(ch->ctx, @@ -993,9 +770,7 @@ void cio_file_close(struct cio_chunk *ch, int delete) } /* Close file descriptor */ - if (cf->fd > 0) { - close(cf->fd); - } + cio_file_native_close(cf); free(cf->path); free(cf); @@ -1006,8 +781,8 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) int ret; int meta_len; int pre_content; - void *tmp; size_t av_size; + size_t old_size; size_t new_size; struct cio_file *cf; @@ -1041,26 +816,20 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) new_size += cf->realloc_size; } + old_size = cf->alloc_size; new_size = ROUND_UP(new_size, ch->ctx->page_size); ret = cio_file_fs_size_change(cf, new_size); + if (ret == -1) { cio_errno(); cio_log_error(ch->ctx, "[cio_file] error setting new file size on write"); return -1; } - /* OSX mman does not implement mremap or MREMAP_MAYMOVE. */ -#ifndef MREMAP_MAYMOVE - if (munmap(cf->map, cf->alloc_size) == -1) { - cio_errno(); - return -1; - } - tmp = mmap(0, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, cf->fd, 0); -#else - tmp = mremap(cf->map, cf->alloc_size, - new_size, MREMAP_MAYMOVE); -#endif - if (tmp == MAP_FAILED) { + + ret = cio_file_native_remap(cf, new_size); + + if (ret != CIO_OK) { cio_errno(); cio_log_error(ch->ctx, "[cio file] data exceeds available space " @@ -1071,10 +840,7 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) cio_log_debug(ch->ctx, "[cio file] alloc_size from %lu to %lu", - cf->alloc_size, new_size); - - cf->map = tmp; - cf->alloc_size = new_size; + old_size, new_size); } if (ch->ctx->options.flags & CIO_CHECKSUM) { @@ -1189,12 +955,21 @@ int cio_file_sync(struct cio_chunk *ch) int ret; int meta_len; int sync_mode; - void *tmp; + size_t file_size; size_t old_size; size_t av_size; size_t size; - struct stat fst; - struct cio_file *cf = (struct cio_file *) ch->backend; + struct cio_file *cf; + + if (ch == NULL) { + return -1; + } + + cf = (struct cio_file *) ch->backend; + + if (cf == NULL) { + return -1; + } if (cf->flags & CIO_OPEN_RD) { return 0; @@ -1204,9 +979,11 @@ int cio_file_sync(struct cio_chunk *ch) return 0; } - ret = fstat(cf->fd, &fst); - if (ret == -1) { + ret = cio_file_native_get_size(cf, &file_size); + + if (ret != CIO_OK) { cio_errno(); + return -1; } @@ -1226,7 +1003,7 @@ int cio_file_sync(struct cio_chunk *ch) } cf->alloc_size = size; } - else if (cf->alloc_size > fst.st_size) { + else if (cf->alloc_size > file_size) { ret = cio_file_fs_size_change(cf, cf->alloc_size); if (ret == -1) { cio_errno(); @@ -1236,20 +1013,11 @@ int cio_file_sync(struct cio_chunk *ch) } } - /* If the mmap size changed, adjust mapping to the proper size */ if (old_size != cf->alloc_size) { -#ifndef MREMAP_MAYMOVE /* OSX */ - if (munmap(cf->map, old_size) == -1) { - cio_errno(); - return -1; - } - tmp = mmap(0, cf->alloc_size, PROT_READ | PROT_WRITE, MAP_SHARED, - cf->fd, 0); -#else - tmp = mremap(cf->map, old_size, cf->alloc_size, MREMAP_MAYMOVE); -#endif - if (tmp == MAP_FAILED) { + ret = cio_file_native_remap(cf, cf->alloc_size); + + if (ret != CIO_OK) { cio_errno(); cio_log_error(ch->ctx, "[cio file] cannot remap memory: old=%lu new=%lu", @@ -1257,7 +1025,6 @@ int cio_file_sync(struct cio_chunk *ch) cf->alloc_size = old_size; return -1; } - cf->map = tmp; } /* Finalize CRC32 checksum */ @@ -1274,23 +1041,27 @@ int cio_file_sync(struct cio_chunk *ch) } /* Commit changes to disk */ - ret = msync(cf->map, cf->alloc_size, sync_mode); - if (ret == -1) { + ret = cio_file_native_sync(cf, sync_mode); + + if (ret != CIO_OK) { cio_errno(); + return -1; } cf->synced = CIO_TRUE; - ret = fstat(cf->fd, &fst); - if (ret == -1) { + ret = cio_file_update_size(cf); + + if (ret != CIO_OK) { cio_errno(); + return -1; } - cf->fs_size = fst.st_size; cio_log_debug(ch->ctx, "[cio file] synced at: %s/%s", ch->st->name, ch->name); + return 0; } @@ -1300,51 +1071,7 @@ int cio_file_sync(struct cio_chunk *ch) */ int cio_file_fs_size_change(struct cio_file *cf, size_t new_size) { - int ret = -1; - - /* - * fallocate() is not portable an Linux only. Since macOS does not have - * fallocate() we use ftruncate(). - */ -#if defined(CIO_HAVE_FALLOCATE) - if (new_size > cf->alloc_size) { - retry: - - if (cf->allocate_strategy == CIO_FILE_LINUX_FALLOCATE) { - /* - * To increase the file size we use fallocate() since this option - * will send a proper ENOSPC error if the file system ran out of - * space. ftruncate() will not fail and upon memcpy() over the - * mmap area it will trigger a 'Bus Error' crashing the program. - * - * fallocate() is not portable, Linux only. - */ - ret = fallocate(cf->fd, 0, 0, new_size); - if (ret == -1 && errno == EOPNOTSUPP) { - /* - * If fallocate fails with an EOPNOTSUPP try operation using - * posix_fallocate. Required since some filesystems do not support - * the fallocate operation e.g. ext3 and reiserfs. - */ - cf->allocate_strategy = CIO_FILE_LINUX_POSIX_FALLOCATE; - goto retry; - } - } - else if (cf->allocate_strategy == CIO_FILE_LINUX_POSIX_FALLOCATE) { - ret = posix_fallocate(cf->fd, 0, new_size); - } - } - else -#endif - { - ret = ftruncate(cf->fd, new_size); - } - - if (!ret) { - cf->fs_size = new_size; - } - - return ret; + return cio_file_native_resize(cf, new_size); } char *cio_file_hash(struct cio_file *cf) From 97671ef20d16482db9dcb86ef6fa655cb2b05eb7 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sat, 9 Jul 2022 18:35:17 -0300 Subject: [PATCH 02/16] build: added native unix file layer Signed-off-by: Leonardo Alminana --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a4fc2d3..9e66638 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,7 @@ set(src cio_os.c cio_log.c + cio_file.c cio_memfs.c cio_chunk.c cio_meta.c @@ -27,7 +28,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") else() set(src ${src} - cio_file.c + cio_file_unix.c ) endif() From 3d36fec51aaf5a7aaf7e901d15fd2f26c1a87e53 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sat, 9 Jul 2022 18:36:58 -0300 Subject: [PATCH 03/16] file: added native layer prototypes Signed-off-by: Leonardo Alminana --- include/chunkio/cio_file_native.h | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 include/chunkio/cio_file_native.h diff --git a/include/chunkio/cio_file_native.h b/include/chunkio/cio_file_native.h new file mode 100644 index 0000000..002f831 --- /dev/null +++ b/include/chunkio/cio_file_native.h @@ -0,0 +1,37 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Chunk I/O + * ========= + * Copyright 2018 Eduardo Silva + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CIO_FILE_NATIVE_H +#define CIO_FILE_NATIVE_H + +#include + +int cio_file_native_unmap(struct cio_file *cf); +int cio_file_native_map(struct cio_file *cf, size_t map_size); +int cio_file_native_remap(struct cio_file *cf, size_t new_size); +int cio_file_native_lookup_user(char *user, void **result); +int cio_file_native_lookup_group(char *group, void **result); +int cio_file_native_get_size(struct cio_file *cf, size_t *file_size); +int cio_file_native_open(struct cio_ctx *ctx, struct cio_file *cf); +void cio_file_native_close(struct cio_file *cf); +int cio_file_native_delete(struct cio_file *cf); +int cio_file_native_sync(struct cio_file *cf, int sync_mode); +int cio_file_native_resize(struct cio_file *cf, size_t new_size); + +#endif From 27209be21ad80add7fb19db0d3b5eb52344a2a38 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sat, 9 Jul 2022 18:37:15 -0300 Subject: [PATCH 04/16] build: added native unix file layer Signed-off-by: Leonardo Alminana --- src/cio_file_unix.c | 463 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 463 insertions(+) create mode 100644 src/cio_file_unix.c diff --git a/src/cio_file_unix.c b/src/cio_file_unix.c new file mode 100644 index 0000000..25427dd --- /dev/null +++ b/src/cio_file_unix.c @@ -0,0 +1,463 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Chunk I/O + * ========= + * Copyright 2018-2019 Eduardo Silva + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +int cio_file_native_unmap(struct cio_file *cf) +{ + int ret; + + if (cf == NULL) { + return -1; + } + + ret = munmap(cf->map, cf->alloc_size); + + if (ret == 0) + { + cf->map = NULL; + } + + return ret; +} + +int cio_file_native_map(struct cio_file *cf, size_t map_size) +{ + int flags; + + if (cf == NULL) { + return CIO_ERROR; + } + + if (cf->map != NULL) { + return CIO_OK; + } + + if (cf->flags & CIO_OPEN_RW) { + flags = PROT_READ | PROT_WRITE; + } + else if (cf->flags & CIO_OPEN_RD) { + flags = PROT_READ; + } + else { + flags = 0; + } + + cf->map = mmap(0, map_size, flags, MAP_SHARED, cf->fd, 0); + + if (cf->map == MAP_FAILED) { + cio_errno(); + + return CIO_ERROR; + } + + return CIO_OK; +} + +int cio_file_native_remap(struct cio_file *cf, size_t new_size) +{ + int result; + void *tmp; + + result = 0; + +/* OSX mman does not implement mremap or MREMAP_MAYMOVE. */ +#ifndef MREMAP_MAYMOVE + result = cio_file_native_unmap(cf); + + if (result == -1) { + return CIO_ERROR; + } + + tmp = mmap(0, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, cf->fd, 0); +#else + (void) result; + + tmp = mremap(cf->map, cf->alloc_size, new_size, MREMAP_MAYMOVE); +#endif + + if (tmp == MAP_FAILED) { + return CIO_ERROR; + } + + cf->map = tmp; + cf->alloc_size = new_size; + + return CIO_OK; +} + +int cio_file_native_lookup_user(char *user, void **result) +{ + long query_buffer_size; + struct passwd *query_result; + char *query_buffer; + struct passwd passwd_entry; + int api_result; + + if (user == NULL) { + *result = calloc(1, sizeof(uid_t)); + + if (*result == NULL) { + cio_errno(); + + return CIO_ERROR; + } + + **(uid_t **) result = (uid_t) -1; + } + + query_buffer_size = sysconf(_SC_GETPW_R_SIZE_MAX); + + if (query_buffer_size == -1) { + query_buffer_size = 4096 * 10; + } + + query_buffer = calloc(1, query_buffer_size); + + if (query_buffer == NULL) { + return CIO_ERROR; + } + + query_result = NULL; + + api_result = getpwnam_r(user, &passwd_entry, query_buffer, + query_buffer_size, &query_result); + + if (api_result != 0 || query_result == NULL) { + cio_errno(); + + free(query_buffer); + + return CIO_ERROR; + } + + *result = calloc(1, sizeof(uid_t)); + + if (*result == NULL) { + cio_errno(); + + free(query_buffer); + + return CIO_ERROR; + } + + **(uid_t **) result = query_result->pw_uid; + + free(query_buffer); + + return CIO_OK; +} + +int cio_file_native_lookup_group(char *group, void **result) +{ + long query_buffer_size; + struct group *query_result; + char *query_buffer; + struct group group_entry; + int api_result; + + if (group == NULL) { + *result = calloc(1, sizeof(gid_t)); + + if (*result == NULL) { + cio_errno(); + + return CIO_ERROR; + } + + **(gid_t **) result = (gid_t) -1; + } + + query_buffer_size = sysconf(_SC_GETGR_R_SIZE_MAX); + + if (query_buffer_size == -1) { + query_buffer_size = 4096 * 10; + } + + query_buffer = calloc(1, query_buffer_size); + + if (query_buffer == NULL) { + return CIO_ERROR; + } + + query_result = NULL; + + api_result = getgrnam_r(group, &group_entry, query_buffer, + query_buffer_size, &query_result); + + if (api_result != 0 || query_result == NULL) { + cio_errno(); + + free(query_buffer); + + return CIO_ERROR; + } + + *result = calloc(1, sizeof(gid_t)); + + if (*result == NULL) { + cio_errno(); + + free(query_buffer); + + return CIO_ERROR; + } + + **(gid_t **) result = query_result->gr_gid; + + free(query_buffer); + + return CIO_OK; +} + +static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path) +{ + mode_t filesystem_acl; + gid_t numeric_group; + uid_t numeric_user; + char *connector; + int result; + char *group; + char *user; + + numeric_group = -1; + numeric_user = -1; + + if (ctx->processed_user != NULL) { + numeric_user = *(uid_t *) ctx->processed_user; + } + + if (ctx->processed_group != NULL) { + numeric_group = *(gid_t *) ctx->processed_group; + } + + if (numeric_user != -1 || numeric_group != -1) { + result = chown(path, numeric_user, numeric_group); + + if (result == -1) { + cio_errno(); + + user = ctx->options.user; + group = ctx->options.group; + connector = "with group"; + + if (user == NULL) { + user = ""; + connector = ""; + } + + if (group == NULL) { + group = ""; + connector = ""; + } + + cio_log_error(ctx, "cannot change ownership of %s to %s %s %s", + path, user, connector, group); + + return CIO_ERROR; + } + } + + if (ctx->options.chmod != NULL) { + filesystem_acl = strtoul(ctx->options.chmod, NULL, 8); + + result = chmod(path, filesystem_acl); + + if (result == -1) { + cio_errno(); + cio_log_error(ctx, "cannot change acl of %s to %s", + path, ctx->options.user); + + return CIO_ERROR; + } + } + + return CIO_OK; +} + +int cio_file_native_get_size(struct cio_file *cf, size_t *file_size) +{ + int ret; + struct stat st; + + ret = -1; + + if (cf->fd != -1) { + ret = fstat(cf->fd, &st); + } + + if (ret == -1) { + ret = stat(cf->path, &st); + } + + if (ret == -1) { + return CIO_ERROR; + } + + if (file_size != NULL) { + *file_size = st.st_size; + } + + return CIO_OK; +} + +/* Open file system file, set file descriptor and file size */ +int cio_file_native_open(struct cio_ctx *ctx, struct cio_file *cf) +{ + int ret; + + if (cf->map != NULL || cf->fd != -1) { + return -1; + } + + /* Open file descriptor */ + if (cf->flags & CIO_OPEN_RW) { + cf->fd = open(cf->path, O_RDWR | O_CREAT, (mode_t) 0600); + } + else if (cf->flags & CIO_OPEN_RD) { + cf->fd = open(cf->path, O_RDONLY); + } + + if (cf->fd == -1) { + cio_errno(); + cio_log_error(ctx, "cannot open/create %s", cf->path); + + return -1; + } + + ret = apply_file_ownership_and_acl_settings(ctx, cf->path); + + if (ret == CIO_ERROR) { + cio_errno(); + cio_file_native_close(cf); + + return -1; + } + + ret = cio_file_update_size(cf); + + if (ret != CIO_OK) { + cio_file_native_close(cf); + + return -1; + } + + return 0; +} + +void cio_file_native_close(struct cio_file *cf) +{ + if (cf != NULL && cf->fd != 0) { + close(cf->fd); + + cf->fd = -1; + } +} + +int cio_file_native_delete(struct cio_file *cf) +{ + return unlink(cf->path); +} + +int cio_file_native_sync(struct cio_file *cf, int sync_mode) +{ + int result; + + result = msync(cf->map, cf->alloc_size, sync_mode); + + if (result == -1) { + return CIO_ERROR; + } + + return CIO_OK; +} + +int cio_file_native_resize(struct cio_file *cf, size_t new_size) +{ + int ret = -1; + + /* + * fallocate() is not portable an Linux only. Since macOS does not have + * fallocate() we use ftruncate(). + */ +#if defined(CIO_HAVE_FALLOCATE) + if (new_size > cf->alloc_size) { + retry: + + if (cf->allocate_strategy == CIO_FILE_LINUX_FALLOCATE) { + /* + * To increase the file size we use fallocate() since this option + * will send a proper ENOSPC error if the file system ran out of + * space. ftruncate() will not fail and upon memcpy() over the + * mmap area it will trigger a 'Bus Error' crashing the program. + * + * fallocate() is not portable, Linux only. + */ + ret = fallocate(cf->fd, 0, 0, new_size); + if (ret == -1 && errno == EOPNOTSUPP) { + /* + * If fallocate fails with an EOPNOTSUPP try operation using + * posix_fallocate. Required since some filesystems do not support + * the fallocate operation e.g. ext3 and reiserfs. + */ + cf->allocate_strategy = CIO_FILE_LINUX_POSIX_FALLOCATE; + goto retry; + } + } + else if (cf->allocate_strategy == CIO_FILE_LINUX_POSIX_FALLOCATE) { + ret = posix_fallocate(cf->fd, 0, new_size); + } + } + else +#endif + { + ret = ftruncate(cf->fd, new_size); + } + + if (!ret) { + cf->fs_size = new_size; + } + + return ret; +} From a2f7989eea5eca0d63b823de0228466ca476b771 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 10:56:36 -0300 Subject: [PATCH 05/16] file: fixed windows fields and added specific error reporting macros Signed-off-by: Leonardo Alminana --- include/chunkio/cio_file.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/chunkio/cio_file.h b/include/chunkio/cio_file.h index a33040b..8820ff6 100644 --- a/include/chunkio/cio_file.h +++ b/include/chunkio/cio_file.h @@ -40,9 +40,8 @@ struct cio_file { char *path; /* root path + stream */ char *map; /* map of data */ #ifdef _WIN32 - void *h; - crc_t crc_be; - int map_synced; + HANDLE backing_file; + HANDLE backing_mapping; #endif /* cached addr */ char *st_content; @@ -61,7 +60,7 @@ void cio_file_close(struct cio_chunk *ch, int delete); int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count); int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size); int cio_file_sync(struct cio_chunk *ch); -int cio_file_fs_size_change(struct cio_file *cf, size_t new_size); +int cio_file_resize(struct cio_file *cf, size_t new_size); char *cio_file_hash(struct cio_file *cf); void cio_file_hash_print(struct cio_file *cf); void cio_file_calculate_checksum(struct cio_file *cf, crc_t *out); @@ -79,4 +78,7 @@ int cio_file_lookup_user(char *user, void **result); int cio_file_lookup_group(char *group, void **result); int cio_file_update_size(struct cio_file *cf); +#define cio_file_report_runtime_error() { cio_file_native_report_runtime_error(); } +#define cio_file_report_os_error() { cio_file_native_report_os_error(); } + #endif From 11cce87039674b03d7574194932fb218bbc1a393 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 10:57:32 -0300 Subject: [PATCH 06/16] file_native: assorted improvements Signed-off-by: Leonardo Alminana --- include/chunkio/cio_file_native.h | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/include/chunkio/cio_file_native.h b/include/chunkio/cio_file_native.h index 002f831..f5dcd73 100644 --- a/include/chunkio/cio_file_native.h +++ b/include/chunkio/cio_file_native.h @@ -22,14 +22,32 @@ #include + + +#ifdef _WIN32 +#define cio_file_native_is_open(cf) (cf->backing_file != INVALID_HANDLE_VALUE) +#define cio_file_native_is_mapped(cf) (cf->backing_mapping != INVALID_HANDLE_VALUE) +#define cio_file_native_report_runtime_error() { cio_errno(); } +#define cio_file_native_report_os_error() { cio_winapi_error(); } +#else +#define cio_file_native_is_open(cf) (cf->fd != -1) +#define cio_file_native_is_mapped(cf) (cf->map != NULL) +#define cio_file_native_report_runtime_error() { cio_errno(); } +#define cio_file_native_report_os_error() { cio_errno(); } +#endif + +int cio_file_native_apply_acl_and_settings(struct cio_ctx *ctx, struct cio_file *cf); +char *cio_file_native_compose_path(char *root_path, char *stream_name, + char *chunk_name); int cio_file_native_unmap(struct cio_file *cf); int cio_file_native_map(struct cio_file *cf, size_t map_size); int cio_file_native_remap(struct cio_file *cf, size_t new_size); int cio_file_native_lookup_user(char *user, void **result); int cio_file_native_lookup_group(char *group, void **result); int cio_file_native_get_size(struct cio_file *cf, size_t *file_size); -int cio_file_native_open(struct cio_ctx *ctx, struct cio_file *cf); -void cio_file_native_close(struct cio_file *cf); +int cio_file_native_filename_check(char *name); +int cio_file_native_open(struct cio_file *cf); +int cio_file_native_close(struct cio_file *cf); int cio_file_native_delete(struct cio_file *cf); int cio_file_native_sync(struct cio_file *cf, int sync_mode); int cio_file_native_resize(struct cio_file *cf, size_t new_size); From 401ea810516d4b648572c210a8772642e4a3e130 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 10:58:30 -0300 Subject: [PATCH 07/16] file: fixed ingeter size warnings Signed-off-by: Leonardo Alminana --- include/chunkio/cio_file_st.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/chunkio/cio_file_st.h b/include/chunkio/cio_file_st.h index 028922f..a6b4f46 100644 --- a/include/chunkio/cio_file_st.h +++ b/include/chunkio/cio_file_st.h @@ -75,8 +75,8 @@ static inline uint16_t cio_file_st_get_meta_len(char *map) /* Set metadata length */ static inline void cio_file_st_set_meta_len(char *map, uint16_t len) { - map[22] = (uint8_t) len >> 8; - map[23] = (uint8_t) len; + map[22] = (uint8_t) (len >> 8); + map[23] = (uint8_t) (len & 0xFF); } /* Return pointer to start point of metadata */ From c0fcc2f3343a1ccb34a0b30cd8982c158df843d6 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 10:59:48 -0300 Subject: [PATCH 08/16] file: improved platform neutrality and error handling Signed-off-by: Leonardo Alminana --- src/cio_file.c | 357 ++++++++++++++++++++++++++++++------------------- 1 file changed, 217 insertions(+), 140 deletions(-) diff --git a/src/cio_file.c b/src/cio_file.c index 270dc29..bfe46fb 100644 --- a/src/cio_file.c +++ b/src/cio_file.c @@ -26,10 +26,7 @@ #include #include #include -#include #include -#include -#include #include #include @@ -178,6 +175,8 @@ static int cio_file_format_check(struct cio_chunk *ch, crc_t crc_check; crc_t crc; + (void) flags; + p = (unsigned char *) cf->map; /* If the file is empty, put the structure on it */ @@ -316,7 +315,7 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) ret = cio_file_native_get_size(cf, &fs_size); if (ret != CIO_OK) { - cio_errno(); + cio_file_report_os_error(); return CIO_ERROR; } @@ -343,14 +342,16 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) /* For empty files, make room in the file system */ size = ROUND_UP(size, ctx->page_size); - ret = cio_file_fs_size_change(cf, size); - if (ret == -1) { - cio_errno(); + ret = cio_file_resize(cf, size); + + if (ret != CIO_OK) { cio_log_error(ctx, "cannot adjust chunk size '%s' to %lu bytes", cf->path, size); + return CIO_ERROR; } + cio_log_debug(ctx, "%s:%s adjusting size OK", ch->st->name, ch->name); } @@ -397,7 +398,6 @@ static int mmap_file(struct cio_ctx *ctx, struct cio_chunk *ch, size_t size) cio_file_native_unmap(cf); cf->data_size = 0; - cf->alloc_size = 0; return CIO_CORRUPTED; } @@ -506,6 +506,41 @@ size_t cio_file_real_size(struct cio_file *cf) return file_size; } +static int format_acl_error_message(struct cio_ctx *ctx, + struct cio_file *cf, + char *output_buffer, + size_t output_buffer_size) +{ + char *connector; + int result; + char *group; + char *user; + + user = ctx->options.user; + group = ctx->options.group; + connector = "with group"; + + if (user == NULL) { + user = ""; + connector = ""; + } + + if (group == NULL) { + group = ""; + connector = ""; + } + + result = snprintf(output_buffer, output_buffer_size - 1, + "cannot change ownership of %s to %s %s %s", + cf->path, user, connector, group); + + if (result < 0) { + return CIO_ERROR; + } + + return CIO_OK; +} + /* * Open or create a data file: the following behavior is expected depending * of the passed flags: @@ -524,34 +559,22 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, size_t size, int *err) { - int psize; - int ret; - int len; - char *path; - struct stat f_st; + char error_message[256]; + char *path; + int ret; struct cio_file *cf; - len = strlen(ch->name); - if (len == 1 && (ch->name[0] == '.' || ch->name[0] == '/')) { - cio_log_error(ctx, "[cio file] invalid file name"); - return NULL; - } + (void) size; - /* Compose path for the file */ - psize = strlen(ctx->options.root_path) + strlen(st->name) + strlen(ch->name); - psize += 8; + ret = cio_file_native_filename_check(ch->name); + if (ret != CIO_OK) { + cio_log_error(ctx, "[cio file] invalid file name"); - path = malloc(psize); - if (!path) { - cio_errno(); return NULL; } - ret = snprintf(path, psize, "%s/%s/%s", - ctx->options.root_path, st->name, ch->name); - if (ret == -1) { - cio_errno(); - free(path); + path = cio_file_native_compose_path(ctx->options.root_path, st->name, ch->name); + if (path == NULL) { return NULL; } @@ -560,6 +583,7 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, if (!cf) { cio_errno(); free(path); + return NULL; } @@ -572,40 +596,95 @@ struct cio_file *cio_file_open(struct cio_ctx *ctx, cf->map = NULL; ch->backend = cf; +#ifdef _WIN32 + cf->backing_file = INVALID_HANDLE_VALUE; + cf->backing_mapping = INVALID_HANDLE_VALUE; +#endif + #if defined (CIO_HAVE_FALLOCATE) cf->allocate_strategy = CIO_FILE_LINUX_FALLOCATE; #endif /* Should we open and put this file up ? */ ret = open_and_up(ctx); + if (ret == CIO_FALSE) { - /* make sure to set the file size before to return */ - ret = stat(cf->path, &f_st); - if (ret == 0) { - cf->fs_size = f_st.st_size; - } + /* we reached our limit, leave the file 'down' */ + cio_file_update_size(cf); + + /* + * Due to he current resource limiting logic we could + * get to this point without a file existing so we just + * ignore the error. + */ - /* we reached our limit, let the file 'down' */ return cf; } - /* Open file (file descriptor and set file size) */ - ret = cio_file_native_open(ctx, cf); - if (ret == -1) { - cio_file_close(ch, CIO_FALSE); - *err = CIO_ERROR; + /* Open the file */ + ret = cio_file_native_open(cf); + + if (ret != CIO_OK) { + free(path); + free(cf); + + *err = ret; + + return NULL; + } + + /* Update the file size field */ + ret = cio_file_update_size(cf); + + if (ret != CIO_OK) { + cio_file_native_close(cf); + + free(path); + free(cf); + + *err = ret; + + return NULL; + } + + /* Set the file ownership and permissions */ + ret = cio_file_native_apply_acl_and_settings(ctx, cf); + + if (ret != CIO_OK) { + *err = ret; + + ret = format_acl_error_message(ctx, cf, error_message, sizeof(error_message)); + + if (ret != CIO_OK) { + cio_log_error(ctx, "error generating error message for acl failure"); + } + else { + cio_log_error(ctx, error_message); + } + + cio_file_native_close(cf); + + free(path); + free(cf); + return NULL; } /* Map the file */ ret = mmap_file(ctx, ch, cf->fs_size); if (ret == CIO_ERROR || ret == CIO_CORRUPTED || ret == CIO_RETRY) { - cio_file_close(ch, CIO_FALSE); + cio_file_native_close(cf); + + free(path); + free(cf); + *err = ret; + return NULL; } *err = CIO_OK; + return cf; } @@ -642,13 +721,19 @@ static int _cio_file_up(struct cio_chunk *ch, int enforced) } /* Open file */ - ret = cio_file_native_open(ch->ctx, cf); - if (ret == -1) { + ret = cio_file_native_open(cf); + + if (ret != CIO_OK) { cio_log_error(ch->ctx, "[cio file] cannot open chunk: %s/%s", ch->st->name, ch->name); return CIO_ERROR; } + ret = cio_file_update_size(cf); + if (ret != CIO_OK) { + return CIO_ERROR; + } + /* * Map content: * @@ -757,21 +842,20 @@ void cio_file_close(struct cio_chunk *ch, int delete) /* Safe unmap of the file content */ munmap_file(ch->ctx, ch); + /* Close file descriptor */ + cio_file_native_close(cf); + /* Should we delete the content from the file system ? */ if (delete == CIO_TRUE) { ret = cio_file_native_delete(cf); - if (ret == -1) { - cio_errno(); + if (ret != CIO_OK) { cio_log_error(ch->ctx, "[cio file] error deleting file at close %s:%s", ch->st->name, ch->name); } } - /* Close file descriptor */ - cio_file_native_close(cf); - free(cf->path); free(cf); } @@ -818,23 +902,12 @@ int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) old_size = cf->alloc_size; new_size = ROUND_UP(new_size, ch->ctx->page_size); - ret = cio_file_fs_size_change(cf, new_size); - - if (ret == -1) { - cio_errno(); - cio_log_error(ch->ctx, - "[cio_file] error setting new file size on write"); - return -1; - } - ret = cio_file_native_remap(cf, new_size); + ret = cio_file_resize(cf, new_size); if (ret != CIO_OK) { - cio_errno(); cio_log_error(ch->ctx, - "[cio file] data exceeds available space " - "(alloc=%lu current_size=%lu write_size=%lu)", - cf->alloc_size, cf->data_size, count); + "[cio_file] error setting new file size on write"); return -1; } @@ -865,8 +938,9 @@ int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size) size_t new_size; size_t content_av; size_t meta_av; - void *tmp; - struct cio_file *cf = ch->backend; + struct cio_file *cf; + + cf = ch->backend; if (cio_file_is_up(ch, cf) == CIO_FALSE) { return -1; @@ -904,33 +978,13 @@ int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size) /* If there is no enough space, increase the file size and it memory map */ if (content_av < size) { new_size = (size - meta_av) + cf->data_size + CIO_FILE_HEADER_MIN; - /* OSX mman does not implement mremap or MREMAP_MAYMOVE. */ -#ifndef MREMAP_MAYMOVE - if (munmap(cf->map, cf->alloc_size) == -1) { - cio_errno(); - return -1; - } - tmp = mmap(0, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, cf->fd, 0); -#else - /* Increase memory map size */ - tmp = mremap(cf->map, cf->alloc_size, new_size, MREMAP_MAYMOVE); -#endif - if (tmp == MAP_FAILED) { - cio_errno(); - cio_log_error(ch->ctx, - "[cio meta] data exceeds available space " - "(alloc=%lu current_size=%lu meta_size=%lu)", - cf->alloc_size, cf->data_size, size); - return -1; - } - cf->map = tmp; - cf->alloc_size = new_size; + ret = cio_file_resize(cf, new_size); + + if (ret != CIO_OK) { + cio_log_error(ch->ctx, + "[cio meta] error resizing mapped file"); - /* Alter file size (file system) */ - ret = cio_file_fs_size_change(cf, new_size); - if (ret == -1) { - cio_errno(); return -1; } } @@ -954,11 +1008,9 @@ int cio_file_sync(struct cio_chunk *ch) { int ret; int meta_len; - int sync_mode; + size_t desired_size; size_t file_size; - size_t old_size; size_t av_size; - size_t size; struct cio_file *cf; if (ch == NULL) { @@ -982,48 +1034,33 @@ int cio_file_sync(struct cio_chunk *ch) ret = cio_file_native_get_size(cf, &file_size); if (ret != CIO_OK) { - cio_errno(); + cio_file_report_os_error(); return -1; } - /* Save current mmap size */ - old_size = cf->alloc_size; - /* If there are extra space, truncate the file size */ av_size = get_available_size(cf, &meta_len); + if (av_size > 0) { - size = cf->alloc_size - av_size; - ret = cio_file_fs_size_change(cf, size); - if (ret == -1) { - cio_errno(); - cio_log_error(ch->ctx, - "[cio file sync] error adjusting size at: " - " %s/%s", ch->st->name, ch->name); - } - cf->alloc_size = size; + desired_size = cf->alloc_size - av_size; } else if (cf->alloc_size > file_size) { - ret = cio_file_fs_size_change(cf, cf->alloc_size); - if (ret == -1) { - cio_errno(); - cio_log_error(ch->ctx, - "[cio file sync] error adjusting size at: " - " %s/%s", ch->st->name, ch->name); - } + desired_size = cf->alloc_size; + } + else { + desired_size = file_size; } - /* If the mmap size changed, adjust mapping to the proper size */ - if (old_size != cf->alloc_size) { - ret = cio_file_native_remap(cf, cf->alloc_size); + if (desired_size != file_size) { + ret = cio_file_resize(cf, desired_size); if (ret != CIO_OK) { - cio_errno(); cio_log_error(ch->ctx, - "[cio file] cannot remap memory: old=%lu new=%lu", - old_size, cf->alloc_size); - cf->alloc_size = old_size; - return -1; + "[cio file sync] error adjusting size at: " + " %s/%s", ch->st->name, ch->name); + + return ret; } } @@ -1032,20 +1069,10 @@ int cio_file_sync(struct cio_chunk *ch) finalize_checksum(cf); } - /* Sync mode */ - if (ch->ctx->options.flags & CIO_FULL_SYNC) { - sync_mode = MS_SYNC; - } - else { - sync_mode = MS_ASYNC; - } - /* Commit changes to disk */ - ret = cio_file_native_sync(cf, sync_mode); + ret = cio_file_native_sync(cf, ch->ctx->options.flags); if (ret != CIO_OK) { - cio_errno(); - return -1; } @@ -1054,8 +1081,6 @@ int cio_file_sync(struct cio_chunk *ch) ret = cio_file_update_size(cf); if (ret != CIO_OK) { - cio_errno(); - return -1; } @@ -1065,13 +1090,64 @@ int cio_file_sync(struct cio_chunk *ch) return 0; } -/* - * Change the size of 'file' in the file system (not memory map). This function - * MUST honor the required new size. - */ -int cio_file_fs_size_change(struct cio_file *cf, size_t new_size) +int cio_file_resize(struct cio_file *cf, size_t new_size) { - return cio_file_native_resize(cf, new_size); + int inner_result; + size_t mapped_size; + int mapped_flag; + int result; + + mapped_flag = cio_file_native_is_mapped(cf); + mapped_size = cf->alloc_size; + +#ifdef _WIN32 + if (mapped_flag) { + result = cio_file_native_unmap(cf); + + if (result != CIO_OK) { + cio_file_native_report_error(); + + return result; + } + } +#endif + + result = cio_file_native_resize(cf, new_size); + + if (result != CIO_OK) { + cio_file_native_report_os_error(); + +#ifdef _WIN32 + if (mapped_flag) { + inner_result = cio_file_native_map(cf, mapped_size); + + if (inner_result != CIO_OK) { + cio_file_native_report_error(); + } + } +#endif + + return result; + } + + if (mapped_flag) { +#ifdef _WIN32 + result = cio_file_native_map(cf, new_size); +#else + result = cio_file_native_remap(cf, new_size); +#endif + + if (result != CIO_OK) { + cio_file_native_report_os_error(); + + return result; + } + } + + (void) mapped_size; + (void) inner_result; + + return CIO_OK; } char *cio_file_hash(struct cio_file *cf) @@ -1138,7 +1214,7 @@ void cio_file_scan_dump(struct cio_ctx *ctx, struct cio_stream *st) (uint32_t) crc_fs, (uint32_t) crc); } } - printf("meta_len=%d, data_size=%lu, crc=%08x\n", + printf("meta_len=%d, data_size=%zu, crc=%08x\n", meta_len, cf->data_size, (uint32_t) crc_fs); if (set_down == CIO_TRUE) { @@ -1152,7 +1228,8 @@ int cio_file_is_up(struct cio_chunk *ch, struct cio_file *cf) { (void) ch; - if (cf->map && cf->fd > 0) { + if (cio_file_native_is_open(cf) && + cio_file_native_is_mapped(cf)) { return CIO_TRUE; } From 60cd239d3802db2a2ee2ce0455f4e98eed7f22f3 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:00:55 -0300 Subject: [PATCH 09/16] file: improved error handling and result type consistency Signed-off-by: Leonardo Alminana --- src/cio_file_unix.c | 220 +++++++++++++++++++++++++++++--------------- 1 file changed, 146 insertions(+), 74 deletions(-) diff --git a/src/cio_file_unix.c b/src/cio_file_unix.c index 25427dd..100c061 100644 --- a/src/cio_file_unix.c +++ b/src/cio_file_unix.c @@ -49,17 +49,25 @@ int cio_file_native_unmap(struct cio_file *cf) int ret; if (cf == NULL) { - return -1; + return CIO_ERROR; + } + + if (!cio_file_native_is_mapped(cf)) { + return CIO_OK; } ret = munmap(cf->map, cf->alloc_size); - if (ret == 0) - { - cf->map = NULL; + if (ret != 0) { + cio_file_native_report_os_error(); + + return CIO_ERROR; } - return ret; + cf->alloc_size = 0; + cf->map = NULL; + + return CIO_OK; } int cio_file_native_map(struct cio_file *cf, size_t map_size) @@ -70,7 +78,11 @@ int cio_file_native_map(struct cio_file *cf, size_t map_size) return CIO_ERROR; } - if (cf->map != NULL) { + if (!cio_file_native_is_open(cf)) { + return CIO_ERROR; + } + + if (cio_file_native_is_mapped(cf)) { return CIO_OK; } @@ -81,17 +93,19 @@ int cio_file_native_map(struct cio_file *cf, size_t map_size) flags = PROT_READ; } else { - flags = 0; + return CIO_ERROR; } cf->map = mmap(0, map_size, flags, MAP_SHARED, cf->fd, 0); if (cf->map == MAP_FAILED) { - cio_errno(); + cio_file_native_report_os_error(); return CIO_ERROR; } + cf->alloc_size = map_size; + return CIO_OK; } @@ -107,7 +121,7 @@ int cio_file_native_remap(struct cio_file *cf, size_t new_size) result = cio_file_native_unmap(cf); if (result == -1) { - return CIO_ERROR; + return result; } tmp = mmap(0, new_size, PROT_READ | PROT_WRITE, MAP_SHARED, cf->fd, 0); @@ -118,6 +132,8 @@ int cio_file_native_remap(struct cio_file *cf, size_t new_size) #endif if (tmp == MAP_FAILED) { + cio_file_native_report_os_error(); + return CIO_ERROR; } @@ -139,7 +155,7 @@ int cio_file_native_lookup_user(char *user, void **result) *result = calloc(1, sizeof(uid_t)); if (*result == NULL) { - cio_errno(); + cio_file_native_report_runtime_error(); return CIO_ERROR; } @@ -165,7 +181,7 @@ int cio_file_native_lookup_user(char *user, void **result) query_buffer_size, &query_result); if (api_result != 0 || query_result == NULL) { - cio_errno(); + cio_file_native_report_os_error(); free(query_buffer); @@ -175,7 +191,7 @@ int cio_file_native_lookup_user(char *user, void **result) *result = calloc(1, sizeof(uid_t)); if (*result == NULL) { - cio_errno(); + cio_file_native_report_runtime_error(); free(query_buffer); @@ -201,7 +217,7 @@ int cio_file_native_lookup_group(char *group, void **result) *result = calloc(1, sizeof(gid_t)); if (*result == NULL) { - cio_errno(); + cio_file_native_report_runtime_error(); return CIO_ERROR; } @@ -227,7 +243,7 @@ int cio_file_native_lookup_group(char *group, void **result) query_buffer_size, &query_result); if (api_result != 0 || query_result == NULL) { - cio_errno(); + cio_file_native_report_os_error(); free(query_buffer); @@ -237,7 +253,7 @@ int cio_file_native_lookup_group(char *group, void **result) *result = calloc(1, sizeof(gid_t)); if (*result == NULL) { - cio_errno(); + cio_file_native_report_runtime_error(); free(query_buffer); @@ -251,15 +267,12 @@ int cio_file_native_lookup_group(char *group, void **result) return CIO_OK; } -static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path) +int cio_file_native_apply_acl_and_settings(struct cio_ctx *ctx, struct cio_file *cf) { mode_t filesystem_acl; gid_t numeric_group; uid_t numeric_user; - char *connector; int result; - char *group; - char *user; numeric_group = -1; numeric_user = -1; @@ -273,27 +286,10 @@ static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path } if (numeric_user != -1 || numeric_group != -1) { - result = chown(path, numeric_user, numeric_group); + result = chown(cf->path, numeric_user, numeric_group); if (result == -1) { - cio_errno(); - - user = ctx->options.user; - group = ctx->options.group; - connector = "with group"; - - if (user == NULL) { - user = ""; - connector = ""; - } - - if (group == NULL) { - group = ""; - connector = ""; - } - - cio_log_error(ctx, "cannot change ownership of %s to %s %s %s", - path, user, connector, group); + cio_file_native_report_os_error(); return CIO_ERROR; } @@ -302,12 +298,13 @@ static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path if (ctx->options.chmod != NULL) { filesystem_acl = strtoul(ctx->options.chmod, NULL, 8); - result = chmod(path, filesystem_acl); + result = chmod(cf->path, filesystem_acl); if (result == -1) { - cio_errno(); + cio_file_native_report_os_error(); + cio_log_error(ctx, "cannot change acl of %s to %s", - path, ctx->options.user); + cf->path, ctx->options.user); return CIO_ERROR; } @@ -323,7 +320,7 @@ int cio_file_native_get_size(struct cio_file *cf, size_t *file_size) ret = -1; - if (cf->fd != -1) { + if (cio_file_native_is_open(cf)) { ret = fstat(cf->fd, &st); } @@ -342,13 +339,63 @@ int cio_file_native_get_size(struct cio_file *cf, size_t *file_size) return CIO_OK; } -/* Open file system file, set file descriptor and file size */ -int cio_file_native_open(struct cio_ctx *ctx, struct cio_file *cf) +char *cio_file_native_compose_path(char *root_path, char *stream_name, + char *chunk_name) { + size_t psize; + char *path; int ret; - if (cf->map != NULL || cf->fd != -1) { - return -1; + /* Compose path for the file */ + psize = strlen(root_path) + + strlen(stream_name) + + strlen(chunk_name) + + 8; + + path = malloc(psize); + + if (path == NULL) { + cio_file_native_report_runtime_error(); + + return NULL; + } + + ret = snprintf(path, psize, "%s/%s/%s", + root_path, stream_name, chunk_name); + + if (ret == -1) { + cio_file_native_report_runtime_error(); + + free(path); + + return NULL; + } + + return path; +} + +int cio_file_native_filename_check(char *name) +{ + size_t len; + + len = strlen(name); + + if (len == 0) { + return CIO_ERROR; + } + if (len == 1) { + if ((name[0] == '.' || name[0] == '/')) { + return CIO_ERROR; + } + } + + return CIO_OK; +} + +int cio_file_native_open(struct cio_file *cf) +{ + if (cio_file_native_is_open(cf)) { + return CIO_OK; } /* Open file descriptor */ @@ -360,53 +407,73 @@ int cio_file_native_open(struct cio_ctx *ctx, struct cio_file *cf) } if (cf->fd == -1) { - cio_errno(); - cio_log_error(ctx, "cannot open/create %s", cf->path); + cio_file_native_report_os_error(); - return -1; + return CIO_ERROR; } - ret = apply_file_ownership_and_acl_settings(ctx, cf->path); + return CIO_OK; +} - if (ret == CIO_ERROR) { - cio_errno(); - cio_file_native_close(cf); +int cio_file_native_close(struct cio_file *cf) +{ + int result; - return -1; + if (cf == NULL) { + return CIO_ERROR; } - ret = cio_file_update_size(cf); + if (cio_file_native_is_open(cf)) { + result = close(cf->fd); - if (ret != CIO_OK) { - cio_file_native_close(cf); + if (result == -1) { + cio_file_native_report_os_error(); - return -1; + return CIO_ERROR; + } + + cf->fd = -1; } - return 0; + return CIO_OK; } -void cio_file_native_close(struct cio_file *cf) +int cio_file_native_delete(struct cio_file *cf) { - if (cf != NULL && cf->fd != 0) { - close(cf->fd); + int result; - cf->fd = -1; + if (cio_file_native_is_open(cf) || + cio_file_native_is_mapped(cf)) { + return CIO_ERROR; } -} -int cio_file_native_delete(struct cio_file *cf) -{ - return unlink(cf->path); + result = unlink(cf->path); + + if (result == -1) { + cio_file_native_report_os_error(); + + return CIO_ERROR; + } + + return CIO_OK; } int cio_file_native_sync(struct cio_file *cf, int sync_mode) { int result; + if (sync_mode & CIO_FULL_SYNC) { + sync_mode = MS_SYNC; + } + else { + sync_mode = MS_ASYNC; + } + result = msync(cf->map, cf->alloc_size, sync_mode); if (result == -1) { + cio_file_native_report_os_error(); + return CIO_ERROR; } @@ -415,7 +482,9 @@ int cio_file_native_sync(struct cio_file *cf, int sync_mode) int cio_file_native_resize(struct cio_file *cf, size_t new_size) { - int ret = -1; + int result; + + result = -1; /* * fallocate() is not portable an Linux only. Since macOS does not have @@ -434,8 +503,8 @@ int cio_file_native_resize(struct cio_file *cf, size_t new_size) * * fallocate() is not portable, Linux only. */ - ret = fallocate(cf->fd, 0, 0, new_size); - if (ret == -1 && errno == EOPNOTSUPP) { + result = fallocate(cf->fd, 0, 0, new_size); + if (result == -1 && errno == EOPNOTSUPP) { /* * If fallocate fails with an EOPNOTSUPP try operation using * posix_fallocate. Required since some filesystems do not support @@ -446,18 +515,21 @@ int cio_file_native_resize(struct cio_file *cf, size_t new_size) } } else if (cf->allocate_strategy == CIO_FILE_LINUX_POSIX_FALLOCATE) { - ret = posix_fallocate(cf->fd, 0, new_size); + result = posix_fallocate(cf->fd, 0, new_size); } } else #endif { - ret = ftruncate(cf->fd, new_size); + result = ftruncate(cf->fd, new_size); } - if (!ret) { + if (result) { + cio_file_native_report_os_error(); + } + else { cf->fs_size = new_size; } - return ret; + return result; } From 62190e4ae8612d8502aaf526cac57ae6f3d73180 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:02:31 -0300 Subject: [PATCH 10/16] file: win32: implemented new set of platform specific primitives and transitioned to mapped files Signed-off-by: Leonardo Alminana --- src/cio_file_win32.c | 1105 ++++++++++++------------------------------ 1 file changed, 297 insertions(+), 808 deletions(-) diff --git a/src/cio_file_win32.c b/src/cio_file_win32.c index 49e8072..5feb06c 100644 --- a/src/cio_file_win32.c +++ b/src/cio_file_win32.c @@ -25,879 +25,444 @@ #include #include #include +#include #include #include #include -/* - * Implement file chunk API for Windows. - * - * * This module uses "files" as shared buffers in lieu of mmap(2). - * - * * Every chunk operation is done directly to the underlying file - * through ReadFile/WriteFile API. - * - * * In this module, cf->map is a plain buffer allocated by malloc. - * cio_file_read_prepare() is in charge of keeping its content - * in sync with the underlying file. - * - * NOTE: there is a shared memory API named CreateFileMapping(), - * but it's not usable in this module. - * - * The reason is that a file mapping prevents file resizes. Since - * there can be several cio_file instances on the same file (see - * cio_scan_stream_files() in cio_scan.c), it ends up preventing - * everyone from resizing that file. - */ - -#define win32_chunk_error(ch, msg) \ - cio_log_error((ch)->ctx, "%s on '%s/%s' (%s() line=%i)", \ - (msg), (ch)->st->name, (ch)->name, __func__, __LINE__) - -static char init_bytes[] = { - /* file type (2 bytes) */ - CIO_FILE_ID_00, CIO_FILE_ID_01, - - /* crc32 (4 bytes) in network byte order */ - 0x41, 0xd9, 0x12, 0xff, - - /* padding bytes (we have 16 extra bytes */ - 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, - - /* metadata length (2 bytes) */ - 0x00, 0x00 -}; - -/* - * Read N bytes from file. Return 0 on success, and -1 on - * failure (including a partial read). - */ -static int read_file(HANDLE h, char *buf, uint64_t size) +int cio_file_native_unmap(struct cio_file *cf) { - uint64_t read = 0; - DWORD len; - DWORD bytes; - - while (read < size) { - len = (int) min(size - read, 65536); - if (!ReadFile(h, buf + read, len, &bytes, NULL)) { - cio_winapi_error(); - return -1; - } - if (bytes == 0) { - cio_winapi_error(); - return -1; /* EOF */ - } - read += bytes; - } - return 0; -} + int result; -static int write_file(HANDLE h, const char *buf, uint64_t size) -{ - uint64_t written = 0; - DWORD len; - DWORD bytes; - - while (written < size) { - len = (int) min(size - written, 65536); - if (!WriteFile(h, buf + written, len, &bytes, NULL)) { - cio_winapi_error(); - return -1; - } - written += bytes; + if (cf == NULL) { + return CIO_ERROR; } - return 0; -} - -static int seek_file(HANDLE h, int64_t offset) -{ - LARGE_INTEGER liDistanceToMove; - - liDistanceToMove.QuadPart = offset; - if (!SetFilePointerEx(h, liDistanceToMove, NULL, FILE_BEGIN)) { - cio_winapi_error(); - return -1; + if (!cio_file_native_is_open(cf)) { + return CIO_OK; } - return 0; -} -static int64_t get_file_size(HANDLE h) -{ - LARGE_INTEGER liFileSize; + if (!cio_file_native_is_mapped(cf)) { + return CIO_OK; + } - if (!GetFileSizeEx(h, &liFileSize)) { - cio_winapi_error(); - return -1; - }; - return liFileSize.QuadPart; -} + result = UnmapViewOfFile(cf->map); -/* - * Create an empty file chunk. This function erases all - * the data inside the given chunk. - */ -static int init_chunk(struct cio_chunk *ch) -{ - struct cio_file *cf = (struct cio_file *) ch->backend; + if (result == 0) { + cio_file_native_report_os_error(); - if (seek_file(cf->h, 0)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - return -1; + return CIO_ERROR; } - if (write_file(cf->h, init_bytes, sizeof(init_bytes))) { - win32_chunk_error(ch, "[cio file] cannot write data"); - return -1; - } + CloseHandle(cf->backing_mapping); - if (!SetEndOfFile(cf->h)) { - cio_winapi_error(); - return -1; - } + cf->backing_mapping = INVALID_HANDLE_VALUE; + cf->alloc_size = 0; + cf->map = NULL; - cf->fs_size = sizeof(init_bytes); - cf->data_size = 0; - return 0; + return CIO_OK; } -/* - * Read the meta size field in file. Return an integer between - * [0, 0xffff] on success, and -1 on failure. - */ -static int read_meta_size(struct cio_chunk *ch) +int cio_file_native_map(struct cio_file *cf, size_t map_size) { - uint16_t meta_size_be; - struct cio_file *cf = (struct cio_file *) ch->backend; + DWORD desired_protection; + DWORD desired_access; - if (seek_file(cf->h, CIO_FILE_CONTENT_OFFSET)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - return -1; + if (cf == NULL) { + return CIO_ERROR; } - if (read_file(cf->h, (char *) &meta_size_be, 2)) { - win32_chunk_error(ch, "[cio file] cannot read meta size"); - return -1; + if (!cio_file_native_is_open(cf)) { + return CIO_ERROR; } - return _byteswap_ushort(meta_size_be); -} - -/* - * Compute CRC32 checksum by scanning through the file. - * Return 0 on success, and -1 on error. - */ -static int calc_checksum(struct cio_file *cf, crc_t *out) -{ - char buf[1024]; - int len; - crc_t val = cio_crc32_init(); - uint64_t read = 0; - uint64_t size = 0; - - if (seek_file(cf->h, CIO_FILE_CONTENT_OFFSET)) { - return -1; + if (cio_file_native_is_mapped(cf)) { + return CIO_OK; } - size = cf->fs_size - CIO_FILE_CONTENT_OFFSET; - - while (read < size) { - len = (int) min(size - read, 1024); - if (read_file(cf->h, buf, len)) { - return -1; - } - val = cio_crc32_update(val, buf, len); - read += len; + if (cf->flags & CIO_OPEN_RW) { + desired_protection = PAGE_READWRITE; + desired_access = FILE_MAP_ALL_ACCESS; } - *out = cio_crc32_finalize(val); - return 0; -} - -/* - * Compute CRC32 and compare it against the checksum stored - * in the file. Return 0 on success, -3 on content corruption - * and -1 on error. - */ -static int verify_checksum(struct cio_chunk *ch) -{ - crc_t hash; - crc_t hash_be; - struct cio_file *cf = (struct cio_file *) ch->backend; - - if (calc_checksum(cf, &hash)) { - win32_chunk_error(ch, "[cio file] cannot compute checksum"); - return CIO_ERROR; + else if (cf->flags & CIO_OPEN_RD) { + desired_protection = PAGE_READONLY; + desired_access = FILE_MAP_READ; } - - if (seek_file(cf->h, 2)) { - win32_chunk_error(ch, "[cio file] cannot seek"); + else { return CIO_ERROR; } - /* - * cio_file.c stores CRC32 in big endian order and Windows - * is a little endian. - */ - if (read_file(cf->h, (char *) &hash_be, 4)) { - win32_chunk_error(ch, "[cio file] cannot read hash"); + cf->backing_mapping = CreateFileMappingA(cf->backing_file, NULL, + desired_protection, + 0, 0, NULL); + + if (cf->backing_mapping == NULL) { + cio_file_native_report_os_error(); + return CIO_ERROR; } - if (hash != (crc_t) _byteswap_ulong(hash_be)) { - cio_log_error(ch->ctx, "[cio file] hash does not match (%u != %u)", - hash, _byteswap_ulong(hash_be)); - return CIO_CORRUPTED; - } + cf->map = MapViewOfFile(cf->backing_mapping, desired_access, 0, 0, map_size); - return CIO_OK; -} + if (cf->map == NULL) { + cio_file_native_report_os_error(); -/* - * Return the number of active file chunks. This function is used - * to check "max_chunks_up" limit. - */ -static int count_open_file_chunks(struct cio_ctx *ctx) -{ - int total = 0; - struct mk_list *head; - struct mk_list *f_head; - struct cio_file *file; - struct cio_chunk *ch; - struct cio_stream *stream; - - mk_list_foreach(head, &ctx->streams) { - stream = mk_list_entry(head, struct cio_stream, _head); - - if (stream->type == CIO_STORE_MEM) { - continue; - } + CloseHandle(cf->backing_mapping); - mk_list_foreach(f_head, &stream->chunks) { - ch = mk_list_entry(f_head, struct cio_chunk, _head); - file = (struct cio_file *) ch->backend; + cf->backing_mapping = INVALID_HANDLE_VALUE; - if (cio_file_is_up(NULL, file) == CIO_TRUE) { - total++; - } - } + return CIO_ERROR; } - return total; -} - -static char *create_path(struct cio_chunk *ch) -{ - char *path; - size_t len; - int ret; - - len = strlen(ch->ctx->options.root_path) + strlen(ch->st->name) + strlen(ch->name); - len += 3; - path = calloc(1, len); - if (!path) { - cio_errno(); - return NULL; - } + cf->alloc_size = map_size; - ret = sprintf_s(path, len, "%s\\%s\\%s", - ch->ctx->options.root_path, ch->st->name, ch->name); - if (ret < 0) { - cio_errno(); - free(path); - return NULL; - } - return path; + return CIO_OK; } -static int is_valid_file_name(const char *name) +int cio_file_native_remap(struct cio_file *cf, size_t new_size) { - size_t len; + /* + * There's no reason for this function to exist because in windows + * we need to unmap, resize and then map again so there's no benefit + * from remapping and I'm not implementing a dummy version because I + * don't want anyone to read it and think there are any reasonable use + * cases for it. + */ - len = strlen(name); - if (len == 0) { - return 0; - } - else if (len == 1) { - if (name[0] == '\\' || name[0] == '.' || name[0] == '/') { - return 0; - } - } - return 1; + (void) cf; + (void) new_size; + + return CIO_ERROR; } -/* - * Fetch the file size regardless of if we opened this file or not. - */ -size_t cio_file_real_size(struct cio_file *cf) +static SID *perform_sid_lookup(char *account_name, SID_NAME_USE *result_sid_type) { - int ret; -#ifdef _WIN64 - struct _stat64 st; -#else - struct _stat32 st; -#endif - - /* Store the current real size */ -#ifdef _WIN64 - ret = _stat64(cf->path, &st); -#else - ret = _stat32(cf->path, &st); -#endif + DWORD referenced_domain_name_length; + char referenced_domain_name[256]; + SID *reallocated_sid_buffer; + DWORD sid_buffer_size; + size_t retry_index; + SID *sid_buffer; + SID_NAME_USE sid_type; + int result; - if (ret != 0) { - cio_errno(); - return 0; - } + referenced_domain_name_length = 256; + sid_buffer_size = 256; - return st.st_size; -} + sid_buffer = calloc(1, sid_buffer_size); -/* - * Return a new file chunk instance. This is the starting - * point for manipulating file chunks. - */ -struct cio_file *cio_file_open(struct cio_ctx *ctx, - struct cio_stream *st, - struct cio_chunk *ch, - int flags, - size_t size, - int *err) -{ - struct cio_file *cf; + if (sid_buffer == NULL) { + cio_file_native_report_runtime_error(); - if (!is_valid_file_name(ch->name)) { - win32_chunk_error(ch, "[cio file] invalid file name"); return NULL; } - cf = calloc(1, sizeof(struct cio_file)); - if (!cf) { - cio_errno(); - return NULL; - } + result = 0; + sid_type = SidTypeUnknown; - cf->path = create_path(ch); - if (cf->path == NULL) { - win32_chunk_error(ch, "[cio file] cannot create path"); - free(cf); - return NULL; - } + for (retry_index = 0 ; retry_index < 5 && !result ; retry_index++) { + result = LookupAccountNameA(NULL, + account_name, + sid_buffer, + &sid_buffer_size, + referenced_domain_name, + &referenced_domain_name_length, + &sid_type); - cf->fd = -1; - cf->flags = flags; - cf->realloc_size = 0; - cf->st_content = NULL; - cf->crc_cur = 0; - cf->map = NULL; - cf->h = INVALID_HANDLE_VALUE; - ch->backend = cf; + if (!result) { + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + sid_buffer_size *= 2; - if (count_open_file_chunks(ch->ctx) >= ctx->max_chunks_up) { - cio_log_debug(ch->ctx, "[cio file] create a chunk %s/%s (down)", - st->name, ch->name); - return cf; /* this is how cio_file.c behaves */ - } + reallocated_sid_buffer = realloc(sid_buffer, sid_buffer_size); - if (cio_file_up(ch)) { - win32_chunk_error(ch, "[cio file] cannot activate chunk"); - cio_file_close(ch, CIO_FALSE); - return NULL; - } + if (reallocated_sid_buffer == NULL) { + cio_file_native_report_runtime_error(); - if (ch->ctx->options.flags & CIO_CHECKSUM) { - if (verify_checksum(ch)) { - win32_chunk_error(ch, "[cio file] cannot verify checksum"); - cio_file_close(ch, CIO_FALSE); - return NULL; - } - } + free(sid_buffer); - return cf; -} + return NULL; + } + } + else { + cio_file_native_report_os_error(); -/* - * Deallocate a file chunk instance. - */ -void cio_file_close(struct cio_chunk *ch, int delete) -{ - struct cio_file *cf = (struct cio_file *) ch->backend; + free(sid_buffer); - if (!cf) { - return; + return NULL; + } + } } - if (cio_file_is_up(ch, ch->backend)) { - cio_file_down(ch); + if (result_sid_type != NULL) { + *result_sid_type = sid_type; } - if (delete == CIO_TRUE) { - if (!DeleteFileA(cf->path)) { - cio_winapi_error(); - } - } - free(cf->map); - free(cf->path); - free(cf); + return sid_buffer; } -/* - * Append data into the end of the file chunk. - */ -int cio_file_write(struct cio_chunk *ch, const void *buf, size_t count) +static int perform_entity_lookup(char *name, + void **result, + SID_NAME_USE desired_sid_type) { - struct cio_file *cf = (struct cio_file *) ch->backend; - int meta_size; - - if (count == 0) { - return 0; - } + SID_NAME_USE result_sid_type; - if (!cio_file_is_up(ch, ch->backend)) { - win32_chunk_error(ch, "[cio file] chunk is not up"); - return -1; - } + *result = (void **) perform_sid_lookup(name, &result_sid_type); - meta_size = read_meta_size(ch); - if (meta_size < 0) { - win32_chunk_error(ch, "[cio file] cannot read meta size"); - return -1; + if (*result == NULL) { + return CIO_ERROR; } - if (seek_file(cf->h, CIO_FILE_HEADER_MIN + meta_size + cf->data_size)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - return -1; - } + if (desired_sid_type != result_sid_type) { + free(*result); - if (write_file(cf->h, buf, count)) { - win32_chunk_error(ch, "[cio file] cannot write data"); - return -1; - } + *result = NULL; - if (!SetEndOfFile(cf->h)) { - cio_winapi_error(); - return -1; + return CIO_ERROR; } - cf->data_size += count; - cf->fs_size = CIO_FILE_HEADER_MIN + meta_size + cf->data_size; - cf->synced = CIO_FALSE; - cf->map_synced = CIO_FALSE; - - return 0; + return CIO_OK; } -int create_space_for_meta(struct cio_chunk *ch, uint16_t meta_size) +int cio_file_native_lookup_user(char *user, void **result) { - uint16_t prev_size; - struct cio_file *cf = (struct cio_file *) ch->backend; - char *buf; - char *ptr; - - if (cf->data_size == 0) { - return 0; /* No need for relocation */ - } - - buf = malloc(cf->fs_size); - if (buf == NULL) { - cio_errno(); - return -1; - } + return perform_entity_lookup(user, result, SidTypeUser); +} - if (seek_file(cf->h, 0)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - free(buf); - return -1; - } +int cio_file_native_lookup_group(char *group, void **result) +{ + return perform_entity_lookup(group, result, SidTypeGroup); +} - if (read_file(cf->h, buf, cf->fs_size)) { - win32_chunk_error(ch, "[cio file] cannot read data"); - free(buf); - return -1; - } +static DWORD cio_file_win_chown(char *path, SID *user, SID *group) +{ + int result; - ptr = cio_file_st_get_content(buf); - prev_size = cio_file_st_get_meta_len(buf); + /* Ownership here does not work in the same way it works in unixes + * so specifying both a user and group will end up with the group + * overriding the user if possible which can cause some misunderstandings. + */ - if (prev_size == meta_size) { - free(buf); - return 0; /* nothing to do */ - } + result = ERROR_SUCCESS; - if (seek_file(cf->h, CIO_FILE_HEADER_MIN + meta_size)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - free(buf); - return -1; + if (user != NULL) { + result = SetNamedSecurityInfoA(path, SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION, + user, NULL, NULL, NULL); } - if (write_file(cf->h, ptr, cf->data_size)) { - win32_chunk_error(ch, "[cio file] cannot write data"); - free(buf); - return -1; + if (group != NULL && result == ERROR_SUCCESS) { + result = SetNamedSecurityInfoA(path, SE_FILE_OBJECT, + GROUP_SECURITY_INFORMATION, + group, NULL, NULL, NULL); } - free(buf); - - cf->fs_size = CIO_FILE_HEADER_MIN + meta_size + cf->data_size; - return 0; + return result; } -int cio_file_write_metadata(struct cio_chunk *ch, char *buf, size_t size) +int cio_file_native_apply_acl_and_settings(struct cio_ctx *ctx, struct cio_file *cf) { - struct cio_file *cf = (struct cio_file *) ch->backend; - uint16_t meta_size = (uint16_t) size; - uint16_t meta_size_be = _byteswap_ushort(meta_size); - - if (!cio_file_is_up(ch, cf)) { - win32_chunk_error(ch, "[cio file] chunk is not up"); - return -1; - } - - if (size > UINT16_MAX) { - cio_log_error(ch->ctx, "[cio file] too large meta (%zu bytes) %s:%s", - size, ch->st->name, ch->name); - return -1; - } - - if (create_space_for_meta(ch, meta_size)) { - cio_log_error(ch->ctx, "[cio file] fail to allocate %zu bytes %s:%s", - size, ch->st->name, ch->name); - return -1; - } + int result; - if (seek_file(cf->h, CIO_FILE_CONTENT_OFFSET)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - return -1; - } + if (ctx->processed_user != NULL) { + result = cio_file_win_chown(cf->path, ctx->processed_user, ctx->processed_group); - if (write_file(cf->h, (char *) &meta_size_be, 2)) { - win32_chunk_error(ch, "[cio file] cannot write meta size"); - return -1; - } + if (result != ERROR_SUCCESS) { + cio_file_native_report_os_error(); - if (write_file(cf->h, buf, meta_size)) { - win32_chunk_error(ch, "[cio file] cannot write meta data"); - return -1; + return CIO_ERROR; + } } - cf->synced = CIO_FALSE; - cf->map_synced = CIO_FALSE; - return 0; + return CIO_OK; } -int cio_file_sync(struct cio_chunk *ch) +static int get_file_size_by_handle(struct cio_file *cf, size_t *file_size) { - struct cio_file *cf = (struct cio_file *) ch->backend; - crc_t hash, hash_be; - - if (calc_checksum(cf, &hash)) { - win32_chunk_error(ch, "[cio file] cannot compute checksum"); - return -1; - } + LARGE_INTEGER native_file_size; + int ret; - if (cf->flags & CIO_OPEN_RD) { - cio_log_debug(ch->ctx, "[cio file] chunk '%s:%s' is read only", - ch->st->name, ch->name); - return 0; - } + memset(&native_file_size, 0, sizeof(native_file_size)); - /* Windows is little endian */ - hash_be = _byteswap_ulong(hash); + ret = GetFileSizeEx(cf->backing_file, &native_file_size); - if (seek_file(cf->h, 2)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - return -1; - } - - if (write_file(cf->h, (char *) &hash_be, 4)) { - win32_chunk_error(ch, "[cio file] cannot write hash"); - return -1; - } - - if (!FlushFileBuffers(cf->h)) { - cio_winapi_error(); - return -1; + if (ret == 0) { + return CIO_ERROR; } - /* Reflect checksum to keep map synced */ - if (cf->map) { - memcpy(cf->map + 2, &hash_be, 4); + if (file_size != NULL) { + *file_size = (size_t) native_file_size.QuadPart; } - cf->crc_be = hash_be; - cf->synced = CIO_TRUE; - - cio_log_debug(ch->ctx, "[cio file] synced at: %s/%s", - ch->st->name, ch->name); - return 0; + return CIO_OK; } -int cio_file_fs_size_change(struct cio_file *cf, size_t new_size) +static int get_file_size_by_path(struct cio_file *cf, size_t *file_size) { - if (seek_file(cf->h, new_size)) { - return -1; + int ret; +#ifdef _WIN64 + struct _stat64 st; +#else + struct _stat32 st; +#endif + +#ifdef _WIN64 + ret = _stat64(cf->path, &st); +#else + ret = _stat32(cf->path, &st); +#endif + + if (ret == -1) { + return CIO_ERROR; } - if (!SetEndOfFile(cf->h)) { - cio_winapi_error(); - return -1; + if (file_size != NULL) { + *file_size = st.st_size; } - cf->data_size += (cf->fs_size - new_size); - cf->fs_size = new_size; - return 0; + return CIO_OK; } -/* - * Return the char pointer to the CRC32 field (big endian). - */ -char *cio_file_hash(struct cio_file *cf) +int cio_file_native_get_size(struct cio_file *cf, size_t *file_size) { - return (char *) &cf->crc_be; -} + int ret; -void cio_file_hash_print(struct cio_file *cf) -{ - crc_t hash; + ret = CIO_ERROR; - if (calc_checksum(cf, &hash)) { - printf("failed to compute hash"); - return; + if (cf->backing_file != INVALID_HANDLE_VALUE) { + ret = get_file_size_by_handle(cf, file_size); } - printf("crc =%u\n", hash); - printf("%08x\n", hash); -} + if (ret != CIO_OK) { + ret = get_file_size_by_path(cf, file_size); + } -void cio_file_calculate_checksum(struct cio_file *cf, crc_t *out) -{ - calc_checksum(cf, out); + return ret; } -void cio_file_scan_dump(struct cio_ctx *ctx, struct cio_stream *st) +char *cio_file_native_compose_path(char *root_path, char *stream_name, + char *chunk_name) { - (void *) ctx; - (void *) st; - return; -} + size_t psize; + char *path; + int ret; -/* - * Copy the file content into memory buffer so that the caller - * can access the chunk data. - */ -int cio_file_read_prepare(struct cio_ctx *ctx, struct cio_chunk *ch) -{ - (void *) ctx; - char *buf; - struct cio_file *cf = ch->backend; - int64_t size; + /* Compose path for the file */ + psize = strlen(root_path) + + strlen(stream_name) + + strlen(chunk_name) + + 3; - if (cf->map && cf->map_synced == CIO_TRUE) { - return 0; /* no need to update */ - } + path = malloc(psize); - size = get_file_size(cf->h); - if (size <= 0) { - win32_chunk_error(ch, "[cio file] cannot get file size"); - return -1; - } + if (path == NULL) { + cio_file_native_report_runtime_error(); - buf = malloc(size); - if (buf == NULL) { - cio_errno(); - return -1; + return NULL; } - if (seek_file(cf->h, 0)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - free(buf); - return -1; - } + ret = snprintf(path, psize, "%s\\%s\\%s", + root_path, stream_name, chunk_name); + + if (ret == -1) { + cio_file_native_report_runtime_error(); + + free(path); - if (read_file(cf->h, buf, size)) { - win32_chunk_error(ch, "[cio file] cannot read data"); - free(buf); - return -1; + return NULL; } - free(cf->map); - cf->map = buf; - cf->map_synced = CIO_TRUE; - return 0; + return path; } -int cio_file_content_copy(struct cio_chunk *ch, - void **out_buf, size_t *out_size) +int cio_file_native_filename_check(char *name) { - char *buf; - int ret = -1; - int meta_size; - int set_down = CIO_FALSE; - struct cio_file *cf = ch->backend; - - if (cio_chunk_is_up(ch) == CIO_FALSE) { - ret = cio_chunk_up_force(ch); - if (ret == -1){ - win32_chunk_error(ch, "[cio file] cannot activate chunk"); - return ret; - } - set_down = CIO_TRUE; - } - - meta_size = read_meta_size(ch); - if (meta_size < 0) { - win32_chunk_error(ch, "[cio file] cannot read meta size"); - goto done; - } + size_t len; - buf = calloc(1, cf->data_size + 1); - if (buf == NULL) { - cio_errno(); - goto done; - } + len = strlen(name); - if (seek_file(cf->h, CIO_FILE_HEADER_MIN + meta_size)) { - win32_chunk_error(ch, "[cio file] cannot seek"); - free(buf); - goto done; + if (len == 0) { + return CIO_ERROR; } - - if (read_file(cf->h, buf, cf->data_size)) { - win32_chunk_error(ch, "[cio file] cannot read data"); - free(buf); - goto done; + else if (len == 1) { + if (name[0] == '\\' || name[0] == '.' || name[0] == '/') { + return CIO_ERROR; + } } - *out_buf = buf; - *out_size = cf->data_size; - ret = 0; - -done: - if (set_down == CIO_TRUE) { - cio_chunk_down(ch); - } - return ret; + return CIO_OK; } -int cio_file_is_up(struct cio_chunk *ch, struct cio_file *cf) +int cio_file_native_open(struct cio_file *cf) { - (void) ch; + DWORD creation_disposition; + DWORD desired_access; - if (cf->h != INVALID_HANDLE_VALUE) { - return CIO_TRUE; + if (cio_file_native_is_open(cf)) { + return CIO_OK; } - return CIO_FALSE; -} - -int cio_file_down(struct cio_chunk *ch) -{ - struct cio_file *cf = (struct cio_file *) ch->backend; - - if (!cio_file_is_up(ch, cf)) { - win32_chunk_error(ch, "[cio file] chunk is not up"); - return -1; + if (cf->flags & CIO_OPEN) { + desired_access = GENERIC_READ | GENERIC_WRITE; + creation_disposition = OPEN_ALWAYS; } - - CloseHandle(cf->h); - cf->h = INVALID_HANDLE_VALUE; - return 0; -} - -int cio_file_up(struct cio_chunk *ch) -{ - if (cio_file_is_up(ch, ch->backend)) { - win32_chunk_error(ch, "[cio file] chunk is already up"); - return -1; + else if (cf->flags & CIO_OPEN_RD) { + desired_access = GENERIC_READ; + creation_disposition = OPEN_EXISTING; } - - if (count_open_file_chunks(ch->ctx) >= ch->ctx->max_chunks_up) { - win32_chunk_error(ch, "[cio file] too many open chunks"); - return -1; + else { + return CIO_ERROR; } - return cio_file_up_force(ch); -} -static SID *perform_sid_lookup(char *account_name, SID_NAME_USE *result_sid_type) -{ - DWORD referenced_domain_name_length; - char referenced_domain_name[256]; - SID *reallocated_sid_buffer; - DWORD sid_buffer_size; - size_t retry_index; - SID *sid_buffer; - SID_NAME_USE sid_type; - int result; + cf->backing_file = CreateFileA(cf->path, + desired_access, + FILE_SHARE_DELETE | + FILE_SHARE_READ | + FILE_SHARE_WRITE, + NULL, + creation_disposition, + FILE_ATTRIBUTE_NORMAL, + NULL); - referenced_domain_name_length = 256; - sid_buffer_size = 256; + if (cf->backing_file == INVALID_HANDLE_VALUE) { + cio_file_native_report_os_error(); - sid_buffer = calloc(1, sid_buffer_size); - - if (sid_buffer == NULL) { - cio_winapi_error(); - - return NULL; + return CIO_ERROR; } - result = 0; - - for (retry_index = 0 ; retry_index < 5 && !result ; retry_index++) { - result = LookupAccountNameA(NULL, - account_name, - sid_buffer, - &sid_buffer_size, - referenced_domain_name, - &referenced_domain_name_length, - &sid_type); + return CIO_OK; +} - if (!result) { - if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - sid_buffer_size *= 2; +int cio_file_native_close(struct cio_file *cf) +{ + int result; - reallocated_sid_buffer = realloc(sid_buffer, sid_buffer_size); + if (cf == NULL) { + return CIO_ERROR; + } - if (reallocated_sid_buffer == NULL) { - cio_winapi_error(); - free(sid_buffer); + if (cio_file_native_is_open(cf)) { + result = CloseHandle(cf->backing_file); - return NULL; - } - } - else { - cio_winapi_error(); - free(sid_buffer); + if (result == 0) { + cio_file_native_report_os_error(); - return NULL; - } + return CIO_ERROR; } - } - if (result_sid_type != NULL) { - *result_sid_type = sid_type; + cf->backing_file = INVALID_HANDLE_VALUE; } - return sid_buffer; + return CIO_OK; } -static int cio_file_lookup_entity(char *name, - void **result, - SID_NAME_USE desired_sid_type) +int cio_file_native_delete(struct cio_file *cf) { - SID_NAME_USE result_sid_type; - - *result = (void **) perform_sid_lookup(name, &result_sid_type); + int result; - if (*result == NULL) { - return CIO_ERROR; - } + result = DeleteFileA(cf->path); - if (desired_sid_type != result_sid_type) { - free(*result); - *result = NULL; + if (result == 0) { + cio_file_native_report_os_error(); return CIO_ERROR; } @@ -905,71 +470,23 @@ static int cio_file_lookup_entity(char *name, return CIO_OK; } -int cio_file_lookup_user(char *user, void **result) -{ - return cio_file_lookup_entity(user, result, SidTypeUser); -} - -int cio_file_lookup_group(char *group, void **result) -{ - return cio_file_lookup_entity(group, result, SidTypeGroup); -} - -static DWORD cio_file_win_chown(char *path, SID *user, SID *group) +int cio_file_native_sync(struct cio_file *cf, int sync_mode) { int result; - /* Ownership here does not work in the same way it works in unixes - * so specifying both a user and group will end up with the group - * overriding the user if possible which can cause some misunderstandings. - */ + result = FlushViewOfFile(cf->map, cf->alloc_size); - result = ERROR_SUCCESS; + if (result == 0) { + cio_file_native_report_os_error(); - if (user != NULL) { - result = SetNamedSecurityInfoA(path, SE_FILE_OBJECT, - OWNER_SECURITY_INFORMATION, - user, NULL, NULL, NULL); - } - - if (group != NULL && result == ERROR_SUCCESS) { - result = SetNamedSecurityInfoA(path, SE_FILE_OBJECT, - GROUP_SECURITY_INFORMATION, - group, NULL, NULL, NULL); + return CIO_ERROR; } - return result; -} - -static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path) -{ - char *connector; - int result; - char *group; - char *user; - - if (ctx->processed_user != NULL) { - result = cio_file_win_chown(path, ctx->processed_user, ctx->processed_group); + if (sync_mode & CIO_FULL_SYNC) { + result = FlushFileBuffers(cf->backing_file); - if (result != CIO_OK) { - cio_errno(); - - user = ctx->options.user; - group = ctx->options.group; - connector = "with group"; - - if (user == NULL) { - user = ""; - connector = ""; - } - - if (group == NULL) { - group = ""; - connector = ""; - } - - cio_log_error(ctx, "cannot change ownership of %s to %s %s %s", - path, user, connector, group); + if (result == 0) { + cio_file_native_report_os_error(); return CIO_ERROR; } @@ -978,68 +495,40 @@ static int apply_file_ownership_and_acl_settings(struct cio_ctx *ctx, char *path return CIO_OK; } -int cio_file_up_force(struct cio_chunk *ch) +int cio_file_native_resize(struct cio_file *cf, size_t new_size) { - struct cio_file *cf = (struct cio_file *) ch->backend; - int dwDesiredAccess = 0; - int dwCreationDisposition = 0; - int meta_size; - int64_t size; - int ret; + LARGE_INTEGER movement_distance; + int result; - if (cio_file_is_up(ch, cf)) { - win32_chunk_error(ch, "[cio file] chunk is already up"); - return -1; + if (!cio_file_native_is_open(cf)) { + return CIO_ERROR; } - if (cf->flags & CIO_OPEN) { - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - dwCreationDisposition = OPEN_ALWAYS; - } - else if (cf->flags & CIO_OPEN_RD) { - dwDesiredAccess = GENERIC_READ; - dwCreationDisposition = OPEN_EXISTING; + if (cio_file_native_is_mapped(cf)) { + return CIO_ERROR; } - cf->h = CreateFileA(cf->path, - dwDesiredAccess, - FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL, - dwCreationDisposition, - FILE_ATTRIBUTE_NORMAL, - NULL); - - if (cf->h == INVALID_HANDLE_VALUE) { - cio_winapi_error(); - win32_chunk_error(ch, "[cio file] cannot open"); - return -1; - } + movement_distance.QuadPart = new_size; - ret = apply_file_ownership_and_acl_settings(ch->ctx, cf->path); - if (ret == CIO_ERROR) { - CloseHandle(cf->h); - cf->h = INVALID_HANDLE_VALUE; + result = SetFilePointerEx(cf->backing_file, + movement_distance, + NULL, FILE_BEGIN); - return -1; - } + if (result == 0) { + cio_file_native_report_os_error(); - size = get_file_size(cf->h); - if (size < 0) { - win32_chunk_error(ch, "[cio file] cannot get file size"); - return -1; - } - else if (size == 0) { - return init_chunk(ch); + return CIO_ERROR; } - meta_size = read_meta_size(ch); - if (meta_size < 0) { - win32_chunk_error(ch, "[cio file] cannot read meta size"); - return -1; + result = SetEndOfFile(cf->backing_file); + + if (result == 0) { + cio_file_native_report_os_error(); + + return CIO_ERROR; } - cf->fs_size = size; - cf->data_size = size - meta_size - CIO_FILE_HEADER_MIN; + cf->fs_size = new_size; - return 0; + return CIO_OK; } From 161c8ca5451046c1e9154f91d1d966db3630a070 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:03:18 -0300 Subject: [PATCH 11/16] memfs: fixed compiler warnings Signed-off-by: Leonardo Alminana --- src/cio_memfs.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cio_memfs.c b/src/cio_memfs.c index 12cc6df..d114355 100644 --- a/src/cio_memfs.c +++ b/src/cio_memfs.c @@ -33,6 +33,11 @@ struct cio_memfs *cio_memfs_open(struct cio_ctx *ctx, struct cio_stream *st, { struct cio_memfs *mf; + (void) flags; + (void) ctx; + (void) ch; + (void) st; + mf = calloc(1, sizeof(struct cio_memfs)); if (!mf) { cio_errno(); @@ -134,12 +139,14 @@ void cio_memfs_scan_dump(struct cio_ctx *ctx, struct cio_stream *st) struct cio_memfs *mf; struct cio_chunk *ch; + (void) ctx; + mk_list_foreach(head, &st->chunks) { ch = mk_list_entry(head, struct cio_chunk, _head); mf = ch->backend; snprintf(tmp, sizeof(tmp) -1, "%s/%s", ch->st->name, ch->name); printf(" %-60s", tmp); - printf("meta_len=%i, data_size=%lu\n", mf->meta_len, mf->buf_len); + printf("meta_len=%i, data_size=%zu\n", mf->meta_len, mf->buf_len); } } From 894076b5588d14af0b2c753b923d0995764d8a12 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:04:02 -0300 Subject: [PATCH 12/16] os: fixed compiler warnings Signed-off-by: Leonardo Alminana --- src/cio_os.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cio_os.c b/src/cio_os.c index b368215..6c6bafe 100644 --- a/src/cio_os.c +++ b/src/cio_os.c @@ -53,7 +53,12 @@ int cio_os_isdir(const char *dir) int cio_os_mkpath(const char *dir, mode_t mode) { struct stat st; - char *dup_dir = NULL; + +#ifdef _WIN32 + char path[MAX_PATH]; +#else + char *dup_dir; +#endif if (!dir) { errno = EINVAL; @@ -70,7 +75,7 @@ int cio_os_mkpath(const char *dir, mode_t mode) } #ifdef _WIN32 - char path[MAX_PATH]; + (void) mode; if (_fullpath(path, dir, MAX_PATH) == NULL) { return 1; From f30b3567d922fb3f09829b96844188acb9a61402 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:04:42 -0300 Subject: [PATCH 13/16] utils: implemented recursive deletion in windows Signed-off-by: Leonardo Alminana --- src/cio_utils.c | 106 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/src/cio_utils.c b/src/cio_utils.c index 16d4af3..45cb0ae 100644 --- a/src/cio_utils.c +++ b/src/cio_utils.c @@ -31,6 +31,7 @@ #include #include +#include #include #ifndef _MSC_VER @@ -95,9 +96,112 @@ int cio_utils_recursive_delete(const char *dir) return ret; } #else +static int cio_utils_recursive_delete_handler(const char *path, + size_t current_depth, + size_t depth_limit) +{ + char search_path[MAX_PATH]; + char entry_path[MAX_PATH]; + DWORD target_file_flags; + HANDLE find_file_handle; + WIN32_FIND_DATAA find_file_data; + int error_detected; + DWORD result; + + result = snprintf(search_path, sizeof(search_path) - 1, "%s\\*", path); + + if (result <= 0) { + return CIO_ERROR; + } + + find_file_handle = FindFirstFileA(search_path, &find_file_data); + + if (find_file_handle == INVALID_HANDLE_VALUE) { + return CIO_ERROR; + } + + target_file_flags = FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_ARCHIVE; + error_detected = CIO_FALSE; + result = 0; + + do { + if (strcmp(find_file_data.cFileName, ".") != 0 && + strcmp(find_file_data.cFileName, "..") != 0) { + + result = snprintf(entry_path, sizeof(entry_path) - 1, "%s\\%s", path, + find_file_data.cFileName); + + if (result > 0) { + if (find_file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if (current_depth < depth_limit) { + result = (DWORD) cio_utils_recursive_delete_handler(entry_path, + current_depth + 1, + depth_limit); + + if (result != CIO_OK) { + error_detected = CIO_TRUE; + } + } + else { + error_detected = CIO_TRUE; + } + } + else if (find_file_data.dwFileAttributes & target_file_flags) { + result = DeleteFileA(entry_path); + + if (result == 0) { + error_detected = CIO_TRUE; + } + } + + } + else { + error_detected = CIO_TRUE; + } + } + + if (error_detected == CIO_FALSE) { + result = FindNextFile(find_file_handle, &find_file_data); + + if (result == 0) { + result = GetLastError(); + + if (result != ERROR_NO_MORE_FILES) { + error_detected = CIO_TRUE; + } + + break; + } + } + } + while (error_detected == CIO_FALSE); + + FindClose(find_file_handle); + + if (error_detected) { + return CIO_ERROR; + } + + result = RemoveDirectoryA(path); + + if (result == 0) { + return CIO_ERROR; + } + + return CIO_OK; +} + int cio_utils_recursive_delete(const char *dir) { - return -1; + DWORD result; + + result = cio_utils_recursive_delete_handler(dir, 0, 100); + + if (result != CIO_OK) { + return -1; + } + + return 0; } #endif From dd6eecf2705ec20fec713e464032719431105f99 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:05:31 -0300 Subject: [PATCH 14/16] win32: dirent: fixed compiler warnings Signed-off-by: Leonardo Alminana --- src/win32/dirent.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/win32/dirent.c b/src/win32/dirent.c index 156ee5d..6ea57f9 100644 --- a/src/win32/dirent.c +++ b/src/win32/dirent.c @@ -38,7 +38,7 @@ struct CIO_WIN32_DIR { /* * Guess POSIX flle type from Win32 file attributes. */ -static int get_filetype(int dwFileAttributes) +static unsigned char get_filetype(int dwFileAttributes) { if (dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { return DT_DIR; @@ -46,6 +46,7 @@ static int get_filetype(int dwFileAttributes) else if (dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { return DT_LNK; } + return DT_REG; } @@ -55,8 +56,8 @@ static int get_filetype(int dwFileAttributes) static char *create_pattern(const char *path) { char *buf; - int len = strlen(path); - int buflen = len + 3; + size_t len = strlen(path); + size_t buflen = len + 3; buf = malloc(buflen); if (buf == NULL) { From a207035e0a37217906c98d003e0f19bd4f730d05 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 11:45:24 -0300 Subject: [PATCH 15/16] file: fixed laggging deprecated error reporting macro names Signed-off-by: Leonardo Alminana --- src/cio_file.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/cio_file.c b/src/cio_file.c index bfe46fb..4737c9f 100644 --- a/src/cio_file.c +++ b/src/cio_file.c @@ -1105,8 +1105,6 @@ int cio_file_resize(struct cio_file *cf, size_t new_size) result = cio_file_native_unmap(cf); if (result != CIO_OK) { - cio_file_native_report_error(); - return result; } } @@ -1120,10 +1118,6 @@ int cio_file_resize(struct cio_file *cf, size_t new_size) #ifdef _WIN32 if (mapped_flag) { inner_result = cio_file_native_map(cf, mapped_size); - - if (inner_result != CIO_OK) { - cio_file_native_report_error(); - } } #endif @@ -1138,8 +1132,6 @@ int cio_file_resize(struct cio_file *cf, size_t new_size) #endif if (result != CIO_OK) { - cio_file_native_report_os_error(); - return result; } } From cf3f6e71010c058ce3b994254b216874499b1183 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Sun, 10 Jul 2022 13:30:43 -0300 Subject: [PATCH 16/16] crc32: fix corner case in apple silicon where the compiler thinks its building for a big endian core Signed-off-by: Leonardo Alminana --- deps/crc32/crc32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/crc32/crc32.c b/deps/crc32/crc32.c index 6af8e23..58b3912 100644 --- a/deps/crc32/crc32.c +++ b/deps/crc32/crc32.c @@ -350,7 +350,7 @@ crc_t crc_update(crc_t crc, const void *data, size_t data_len) const uint32_t *d32 = (const uint32_t *)d; while (data_len >= 8) { -#if __BYTE_ORDER == __BIG_ENDIAN +#if defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN crc_t d1 = *d32++ ^ le16toh(crc); crc_t d2 = *d32++; crc =