fix: memory overflow (#28)

* temp: find overflow * temp: parker cannot test pass * fix: args->length + 1 not wrapped in brackets
nature-lang · Sep 22, 2023 · ea64e4a · ea64e4a
1 parent c028764
commit ea64e4a
Show file tree

Hide file tree

Showing 28 changed files with 215 additions and 97 deletions.
diff --git a/runtime/allocator.c b/runtime/allocator.c
@@ -40,29 +40,38 @@ static bool summary_find_continuous(uint8_t level, page_summary_t *summaries, ui
     uint64_t find_max = 0;
     for (uint64_t i = *start; i < *end; ++i) {
         page_summary_t s = summaries[i];
-        find_max += s.start;
+        find_max += s.start; // 左侧 + 新的 s 的 start
         if (find_max >= pages_count) {
-            // 找到了
             find = true;
             find_end = i;
             break;
         }
 
+        // 当前 chunk 中间存在更大的 max 区域, 此时从当前 chunk 开始从新计算 find_max
         if (s.max > find_max) {
             find_max = s.max;
             find_start = i;
         }
+
+        // find max 超过了需求熟练
         if (find_max >= pages_count) {
-            // 找到了
+            // 找到了完整的空闲位置
             find = true;
             find_end = i;
             break;
         }
 
-        // 目前的空间都不满足条件，判断当前空间是否是完整空闲，非完整的空闲，必须要中断了
+        // 目前的空间都不满足条件，判断当前空间是否是整个空闲(s.end = max_pages_count 表示全空闲)
+        // 如果非完整的空闲，则会导致连续性中断, 此时从 s.end 从新开始计算
         if (s.end != max_pages_count) {
-            find_start = i;
-            find_max = s.end;
+            if (s.end > 0) {
+                find_start = i; // 已经从 i 开始记录
+                find_max = s.end;
+            } else {
+                // s.end == 0, 表示当前 chunk 完全不可用，此时充值 find_max, 并且更新 find_start 为下一个块
+                find_start = i + 1;
+                find_max = 0;
+            }
         }
     }
     *start = find_start;
@@ -307,6 +316,8 @@ static void chunks_set(addr_t base, uint64_t size, bool v) {
             bit_end = CHUNK_BITS_COUNT - 1;
         }
 
+        DEBUGF("[runtime.chunks_set] chunk index: %lu, chunk block base: %p, bit_start: %lu, bit_end: %lu",
+               index, (void *) chunk->blocks, bit_start, bit_end);
         for (uint64_t i = bit_start; i <= bit_end; ++i) {
             if (v) {
                 bitmap_set((uint8_t *) chunk->blocks, i);
@@ -353,6 +364,8 @@ static addr_t page_alloc_find(uint64_t pages_count) {
         }
     }
 
+    DEBUGF("[runtime.page_alloc_find] find continuous pages, start: %lu, end: %lu", start, end);
+
     // start ~ end 指向的一组 chunks 中包含连续的内存空间，现在需要确认起起点位置(假设 start == end, 其 可能是 start or mid or end 中的任意一个位置)
     addr_t find_addr = 0;
     if (start == end) {
@@ -378,19 +391,28 @@ static addr_t page_alloc_find(uint64_t pages_count) {
         // 计算 find_addr
         find_addr = chunk_base(start) + bit_start * ALLOC_PAGE_SIZE;
 
+        DEBUGF("[runtime.page_alloc_find] find addr=%p, start == end, start: %lu, chunk_base: %p, bit start: %lu, bit end: %lu",
+               (void *) find_addr, start, chunk->blocks, bit_start, bit_end);
+
         // 更新从 find_addr 对应的 bit ~ page_count 位置的所有 chunk 的 bit 为 1
     } else {
-        // 跨越多个块
+        // start ~ end 这一段连续的内存空间跨越多个 chunk
         page_summary_t *l5_summaries = page_alloc->summary[PAGE_SUMMARY_LEVEL - 1];
         page_summary_t start_summary = l5_summaries[start];
         uint64_t bit_start = CHUNK_BITS_COUNT + 1 - start_summary.end;
+        DEBUGF("[runtime.page_alloc_find] find addr=%p, start != end, start chunk: %lu, chunk summary [%d, %d, %d],"
+               " end: %lu, bit start: %lu",
+               (void *) find_addr, start, start_summary.start, start_summary.max, start_summary.end, end, bit_start);
         find_addr = chunk_base(start) + bit_start * ALLOC_PAGE_SIZE;
     }
     assertf(find_addr % ALLOC_PAGE_SIZE == 0, "find addr=%p not align_up", find_addr);
 
     // 更新相关的 chunks 为使用状态
     chunks_set(find_addr, pages_count * ALLOC_PAGE_SIZE, 1);
 
+    DEBUGF("[runtime.page_alloc_find] find_addr: %p, page_count: %lu, size: %lu",
+           (void *) find_addr, pages_count, pages_count * ALLOC_PAGE_SIZE);
+
     return find_addr;
 }
 
@@ -442,6 +464,7 @@ void *mheap_sys_alloc(mheap_t *mheap, uint64_t *size) {
 
     void *v = NULL;
     while (true) {
+        allocated_total_bytes += alloc_size;
         v = sys_memory_map((void *) hint->addr, alloc_size);
         if (v == (void *) hint->addr) {
             // 分配成功, 定义下一个分配点,基于此可以获得 64MB 对齐的内存地址
@@ -465,6 +488,9 @@ void *mheap_sys_alloc(mheap_t *mheap, uint64_t *size) {
         slice_push(mheap->arena_indexes, (void *) i);
     }
 
+    // 虚拟内存映射并不会真的写入到内存，必须触发内存页中断才行
+    DEBUGF("[mheap_sys_alloc] allocate_total_bytes=%lu", allocated_total_bytes);
+
     *size = alloc_size;
     return v;
 }
@@ -638,9 +664,12 @@ static mspan_t *mcache_refill(mcache_t *mcache, uint64_t spanclass) {
         uncache_span(mcentral, mspan);
     }
 
-    // cache
+    // 从 mcentral 中读取一个 span 进行读取
     mspan = cache_span(mcentral);
     mcache->alloc[spanclass] = mspan;
+    DEBUGF("[runtime.mcache_refill] mcentral=%p, spanclass=%lu|%d, mspan_base=%p - %p, obj_size=%lu, alloc_count=%lu",
+           mcentral, spanclass, mspan->spanclass, (void *) mspan->base, (void *) mspan->end, mspan->obj_size,
+           mspan->alloc_count);
     return mspan;
 }
 
@@ -824,7 +853,15 @@ void mheap_free_span(mheap_t *mheap, mspan_t *span) {
     // arena.bits 保存了当前 span 中的指针 bit, 当下一次当前内存被分配时会覆盖写入
     // 垃圾回收期间不会有任何指针指向该空间，因为当前 span 就是因为没有被任何 ptr 指向才被回收的
 
+    remove_total_bytes += span->pages_count * ALLOC_PAGE_SIZE;
+
     // 将物理内存归还给操作系统
+    DEBUGF("[runtime.mheap_free_span] remove_total_bytes=%lu MB, span.base=0x%lx, span.pages_count=%ld, remove_size=%lu",
+           remove_total_bytes / 1024 / 1024,
+           span->base,
+           span->pages_count,
+           span->pages_count * ALLOC_PAGE_SIZE);
+
     sys_memory_remove((void *) span->base, span->pages_count * ALLOC_PAGE_SIZE);
 }
 

diff --git a/runtime/collector.c b/runtime/collector.c
@@ -267,14 +267,9 @@ static void sweep_span(linked_t *full, linked_t *partial, mspan_t *span) {
             // 内存回收(未返回到堆)
             allocated_bytes -= span->obj_size;
 
-#ifdef DEBUG
             DEBUGF("[runtime.sweep_span] success, span->class=%d, span->base=0x%lx, span->obj_size=%ld, obj_addr=0x%lx, allocator_bytes=%ld",
                    span->spanclass, span->base, span->obj_size, span->base + i * span->obj_size, allocated_bytes);
 
-            // 将对应位置的内存设置为 0
-            memset((void *) (span->base + i * span->obj_size), 0, span->obj_size);
-#endif
-
         }
     }
 
@@ -287,7 +282,7 @@ static void sweep_span(linked_t *full, linked_t *partial, mspan_t *span) {
     span->alloc_count = bitmap_set_count(span->alloc_bits);
 
     if (span->alloc_count == 0) {
-        DEBUGF("span free to heap, base=0x%lx, class=%d", span->base, span->spanclass);
+        DEBUGF("[sweep_span] span free to heap, base=0x%lx, class=%d", span->base, span->spanclass);
         mheap_free_span(memory->mheap, span);
         free_mspan_meta(span);
         return;

diff --git a/runtime/memory.c b/runtime/memory.c
@@ -1,11 +1,17 @@
 #include "memory.h"
+#include "stdlib.h"
 
+uint64_t remove_total_bytes = 0; // 当前回收到物理内存中的总空间
+uint64_t allocated_total_bytes = 0; // 当前分配的总空间
 uint64_t allocated_bytes = 0; // 当前分配的内存空间
 uint64_t next_gc_bytes = 0; // 下一次 gc 的内存量
 bool force_gc = 0; // runtime_judge_gc 总是立刻进行 gc
 
 rtype_t *rt_find_rtype(uint32_t rtype_hash) {
-    return table_get(rt_rtype_table, itoa(rtype_hash));
+    char *str = itoa(rtype_hash);
+    rtype_t *result = table_get(rt_rtype_table, str);
+    free(str);
+    return result;
 }
 
 uint64_t rt_rtype_out_size(uint32_t rtype_hash) {
@@ -82,8 +88,9 @@ void rtypes_deserialize() {
         }
 
         // rtype 已经组装完毕，现在加入到 rtype table 中
-        table_set(rt_rtype_table, itoa(r->hash), r);
-
+        char *str = itoa(r->hash);
+        table_set(rt_rtype_table, str, r);
+        free(str);
     }
 
 

diff --git a/runtime/memory.h b/runtime/memory.h
@@ -55,6 +55,8 @@
 #define DEFAULT_NEXT_GC_BYTES  (100 * 1024) // 100KB
 #define NEXT_GC_FACTOR 2
 
+extern uint64_t remove_total_bytes; // 当前回收到物理内存中的总空间
+extern uint64_t allocated_total_bytes; // 当前分配的总空间
 extern uint64_t allocated_bytes; // 当前分配的内存空间
 extern uint64_t next_gc_bytes; // 下一次 gc 的内存量
 extern bool force_gc; // runtime_judge_gc 总是立刻进行 gc

diff --git a/runtime/nutils/array.h b/runtime/nutils/array.h
@@ -24,11 +24,13 @@ static inline n_array_t *rt_array_new(rtype_t *element_rtype, uint64_t length) {
            addr,
            element_rtype->size,
            type_kind_str[element_rtype->kind],
-            element_rtype->last_ptr > 0,
+           element_rtype->last_ptr > 0,
            rtype.size,
            length,
            type_kind_str[rtype.kind])
 
+    free(rtype.gc_bits);
+
     return addr;
 }
 

diff --git a/runtime/nutils/hash.h b/runtime/nutils/hash.h
@@ -43,14 +43,19 @@ static inline uint64_t extract_data_index(uint64_t hash_value) {
 
 static inline uint64_t key_hash(rtype_t *rtype, void *key_ref) {
     char *str = rtype_value_str(rtype, key_ref);
-    return hash_string(str);
+    uint64_t result = hash_string(str);
+    free((void *) str);
+    return result;
 }
 
 static inline bool key_equal(rtype_t *rtype, void *actual, void *expect) {
     DEBUGF("[key_equal] actual=%p, expect=%p", actual, expect)
     char *actual_str = rtype_value_str(rtype, actual);
     char *expect_str = rtype_value_str(rtype, expect);
-    return str_equal(actual_str, expect_str);
+    bool result = str_equal(actual_str, expect_str);
+    free((void *) actual_str);
+    free((void *) expect_str);
+    return result;
 }
 
 /**

diff --git a/runtime/nutils/map.c b/runtime/nutils/map.c
@@ -114,6 +114,7 @@ n_cptr_t map_access(n_map_t *m, void *key_ref) {
         return 0;
     }
 
+    free((void *) key_str);
     uint64_t data_index = get_data_index(m, hash_index);
 
     // 找到值所在中数组位置起始点并返回
@@ -144,6 +145,7 @@ n_cptr_t map_assign(n_map_t *m, void *key_ref) {
            key_rtype->kind,
            key_str,
            hash_index);
+    free((void *) key_str);
 
     uint64_t data_index = 0;
     if (hash_value_empty(hash_value)) {

diff --git a/runtime/nutils/set.c b/runtime/nutils/set.c
@@ -43,6 +43,7 @@ static void set_grow(n_set_t *m) {
     DEBUGF("[runtime.set_grow] len=%lu, cap=%lu, key_data=%p, hash_table=%p", m->length, m->capacity, m->key_data,
            m->hash_table);
 
+    assert(m->key_rtype_hash > 0);
     rtype_t *key_rtype = rt_find_rtype(m->key_rtype_hash);
     assertf(key_rtype, "cannot find key_rtype by hash %lu", m->key_rtype_hash);
     uint64_t key_size = rtype_out_size(key_rtype, POINTER_SIZE);

diff --git a/runtime/nutils/string.c b/runtime/nutils/string.c
@@ -44,6 +44,11 @@ n_string_t *string_new(void *raw_string, uint64_t length) {
 void *string_ref(n_string_t *n_str) {
     DEBUGF("[runtime.string_ref] length=%lu, data=%p", n_str->length, n_str->data);
 
+    // 空间足够，且最后一位已经是 0， 可以直接返回
+    if (n_str->capacity > n_str->length && n_str->data[n_str->length] == '\0') {
+        return n_str->data;
+    }
+
     // 结尾添加 '\0' 字符
     int a = '\0';
     vec_push(n_str, &a);

diff --git a/runtime/nutils/syscall.c b/runtime/nutils/syscall.c
@@ -22,12 +22,12 @@ void syscall_exec(n_string_t *path, n_vec_t *argv, n_vec_t *envp) {
     char *p_str = string_ref(path);
 
     // args 转换成 char* 格式并给到 execve
-    char **c_args = mallocz(sizeof(char *) * argv->length + 1);
+    char **c_args = mallocz(sizeof(char *) * (argv->length + 1));
     for (int i = 0; i < argv->length; ++i) {
         n_string_t *arg;
         vec_access(argv, i, &arg);
         if (arg == NULL) {
-            return;
+            continue;
         }
 
         char *c_arg = string_ref(arg);
@@ -36,15 +36,17 @@ void syscall_exec(n_string_t *path, n_vec_t *argv, n_vec_t *envp) {
     c_args[argv->length] = NULL; // 最后一个元素为 NULL
 
     // envs
-    char **c_envs = mallocz(sizeof(char *) * envp->length);
+    DEBUGF("[syscall_exec] envp->length=%lu", envp->length);
+    char **c_envs = mallocz(sizeof(char *) * (envp->length + 1));
     for (int i = 0; i < envp->length; ++i) {
         n_string_t *env;
         vec_access(envp, i, &env);
         if (env == NULL) {
-            return;
+            continue;
         }
 
         char *c_env = string_ref(env);
+        DEBUGF("[syscall_exec] c_env=%p, data=%s, strlen=%lu", (void *) c_env, c_env, strlen(c_env));
         c_envs[i] = c_env;
     }
     c_envs[envp->length] = NULL;

diff --git a/runtime/nutils/vec.c b/runtime/nutils/vec.c
@@ -89,6 +89,7 @@ void vec_assign(n_vec_t *l, uint64_t index, void *ref) {
 
     rtype_t *element_rtype = rt_find_rtype(l->element_rtype_hash);
     uint64_t element_size = rtype_out_size(element_rtype, POINTER_SIZE);
+    DEBUGF("[runtime.vec_assign] element_size=%lu", element_size);
     // 计算 offset
     uint64_t offset = rtype_out_size(element_rtype, POINTER_SIZE) * index; // (size unit byte) * index
     void *p = l->data + offset;

diff --git a/runtime/runtime.c b/runtime/runtime.c
@@ -26,7 +26,13 @@ char *rtype_value_str(rtype_t *rtype, void *data_ref) {
     if (rtype->kind == TYPE_STRING) {
         n_string_t *n_str = (void *) fetch_addr_value((addr_t) data_ref); // 读取栈中存储的值
         assertf(n_str && n_str->length > 0, "fetch addr by data ref '%p' err", data_ref);
-        return (char *) string_ref(n_str);
+
+//        return strdup(string_ref(n_str));
+        // 进行 data copy, 避免被 free
+        char *str = mallocz(n_str->length + 1);
+        memmove(str, n_str->data, n_str->length);
+        str[n_str->length] = '\0';
+        return str;
     }
 
     assertf(false, "not support kind=%s", type_kind_str[rtype->kind]);

diff --git a/src/ssa.c b/src/ssa.c
@@ -31,19 +31,25 @@ static bool self_is_imm_dom(slice_t *be_doms, basic_block_t *self, uint64_t awai
 
 /**
  * 计算 block 中的直接支配者
- * @param be_doms
- * @param block
+ * @param doms
+ * @param self
  * @return
  */
-static basic_block_t *calc_imm_domer(slice_t *be_doms, basic_block_t *await) {
-    for (int i = 0; i < be_doms->count; ++i) {
-        basic_block_t *self = be_doms->take[i];
+static basic_block_t *calc_imm_domer(slice_t *doms, basic_block_t *self) {
+    for (int i = 0; i < doms->count; ++i) {
+        basic_block_t *dom = doms->take[i];
+
+        // self 不能作为自己的的最近支配节点
+        if (dom->id == self->id) {
+            continue;
+        }
+
         // 判断 item 是否被除了 [await->id 和 self->id 以外的所有 id 所支配]
-        if (self_is_imm_dom(be_doms, self, await->id)) {
-            return self;
+        if (self_is_imm_dom(doms, dom, self->id)) {
+            return dom;
         }
     }
-    assertf(false, "block=%s must have one imm dom", await->name);
+    assertf(false, "block=%s must have one imm dom", self->name);
     exit(1);
 }
 
@@ -121,7 +127,7 @@ void ssa_domers(closure_t *c) {
 /**
  * 计算最近支配点
  * 在支配 p 的点中，若一个支配点 i (i != p)， i 被 p 剩下其他的所有的支配点支配，则称 i 为 p 的最近支配点 imm_domer
- * B0 没有 imm_domer
+ * B0 没有 imm_domer, 其他所有节点至少一个一个除了自身意外的最小支配者！
  * imm_domer 一定是父节点中的某一个
  * @param c
  */

diff --git a/tests/blackbox/20230922_00_for.c b/tests/blackbox/20230922_00_for.c
@@ -0,0 +1,15 @@
+#include "tests/test.h"
+#include "utils/assertf.h"
+#include "utils/exec.h"
+#include <stdio.h>
+
+static void test_basic() {
+//    char *raw = exec_output();
+//    printf("%s", raw);
+
+//    assert_string_equal(raw, "hello world 2022 724 11.550000 ! ! !\n");
+}
+
+int main(void) {
+    TEST_BASIC
+}