-
-
Notifications
You must be signed in to change notification settings - Fork 272
/
Copy pathH5Oalloc.c
2497 lines (2117 loc) · 113 KB
/
H5Oalloc.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* [email protected]. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*-------------------------------------------------------------------------
*
* Created: H5Oalloc.c
*
* Purpose: Object header allocation routines.
*
*-------------------------------------------------------------------------
*/
/****************/
/* Module Setup */
/****************/
#include "H5Omodule.h" /* This source code file is part of the H5O module */
/***********/
/* Headers */
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5FLprivate.h" /* Free lists */
#include "H5MFprivate.h" /* File memory management */
#include "H5MMprivate.h" /* Memory management */
#include "H5Opkg.h" /* Object headers */
/****************/
/* Local Macros */
/****************/
/******************/
/* Local Typedefs */
/******************/
/********************/
/* Package Typedefs */
/********************/
/********************/
/* Local Prototypes */
/********************/
static herr_t H5O__add_gap(H5F_t *f, H5O_t *oh, unsigned chunkno, hbool_t *chk_dirtied, size_t idx,
uint8_t *new_gap_loc, size_t new_gap_size);
static herr_t H5O__eliminate_gap(H5O_t *oh, hbool_t *chk_dirtied, H5O_mesg_t *mesg, uint8_t *new_gap_loc,
size_t new_gap_size);
static herr_t H5O__alloc_null(H5F_t *f, H5O_t *oh, size_t null_idx, const H5O_msg_class_t *new_type,
void *new_native, size_t new_size);
static htri_t H5O__alloc_extend_chunk(H5F_t *f, H5O_t *oh, unsigned chunkno, size_t size, size_t *msg_idx);
static herr_t H5O__alloc_find_best_nonnull(const H5F_t *f, const H5O_t *oh, size_t *size,
H5O_msg_alloc_info_t *found_msg);
static herr_t H5O__alloc_new_chunk(H5F_t *f, H5O_t *oh, size_t size, size_t *new_idx);
static herr_t H5O__alloc_find_best_null(const H5O_t *oh, size_t size, size_t *mesg_idx);
static htri_t H5O__move_cont(H5F_t *f, H5O_t *oh, unsigned cont_u);
static htri_t H5O__move_msgs_forward(H5F_t *f, H5O_t *oh);
static htri_t H5O__merge_null(H5F_t *f, H5O_t *oh);
static htri_t H5O__remove_empty_chunks(H5F_t *f, H5O_t *oh);
static herr_t H5O__alloc_shrink_chunk(H5F_t *f, H5O_t *oh, unsigned chunkno);
/*********************/
/* Package Variables */
/*********************/
/* Declare extern the free list for H5O_cont_t's */
H5FL_EXTERN(H5O_cont_t);
/*****************************/
/* Library Private Variables */
/*****************************/
/*******************/
/* Local Variables */
/*******************/
/*-------------------------------------------------------------------------
* Function: H5O__add_gap
*
* Purpose: Add a gap to a chunk
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O__add_gap(H5F_t H5_ATTR_NDEBUG_UNUSED *f, H5O_t *oh, unsigned chunkno, hbool_t *chk_dirtied, size_t idx,
uint8_t *new_gap_loc, size_t new_gap_size)
{
hbool_t merged_with_null; /* Whether the gap was merged with a null message */
size_t u; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* check args */
assert(oh);
assert(oh->version > H5O_VERSION_1);
assert(chk_dirtied);
assert(new_gap_loc);
assert(new_gap_size);
#ifndef NDEBUG
if (chunkno > 0) {
unsigned chk_proxy_status = 0; /* Object header chunk proxy entry cache status */
/* Check the object header chunk proxy's status in the metadata cache */
if (H5AC_get_entry_status(f, oh->chunk[chunkno].addr, &chk_proxy_status) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL,
"unable to check metadata cache status for object header chunk proxy");
/* Make certain that object header is protected */
assert(chk_proxy_status & H5AC_ES__IS_PROTECTED);
} /* end if */
#endif /* NDEBUG */
/* Check for existing null message in chunk */
merged_with_null = FALSE;
for (u = 0; u < oh->nmesgs && !merged_with_null; u++) {
/* Find a null message in the chunk with the new gap */
/* (a null message that's not the one we are eliminating) */
if (H5O_NULL_ID == oh->mesg[u].type->id && oh->mesg[u].chunkno == chunkno && u != idx) {
/* Sanity check - chunks with null messages shouldn't have a gap */
assert(oh->chunk[chunkno].gap == 0);
/* Eliminate the gap in the chunk */
if (H5O__eliminate_gap(oh, chk_dirtied, &oh->mesg[u], new_gap_loc, new_gap_size) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't eliminate gap in chunk");
/* Set flag to indicate that the gap was handled */
merged_with_null = TRUE;
} /* end if */
} /* end for */
/* If we couldn't find a null message in the chunk, move the gap to the end */
if (!merged_with_null) {
/* Adjust message offsets after new gap forward in chunk */
for (u = 0; u < oh->nmesgs; u++)
if (oh->mesg[u].chunkno == chunkno && oh->mesg[u].raw > new_gap_loc)
oh->mesg[u].raw -= new_gap_size;
/* Slide raw message info forward in chunk image */
memmove(new_gap_loc, new_gap_loc + new_gap_size,
(size_t)((oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh))) -
(new_gap_loc + new_gap_size)));
/* Add existing gap size to new gap size */
new_gap_size += oh->chunk[chunkno].gap;
/* Merging with existing gap will allow for a new null message */
if (new_gap_size >= (size_t)H5O_SIZEOF_MSGHDR_OH(oh)) {
H5O_mesg_t *null_msg; /* Pointer to new null message */
/* Check if we need to extend message table to hold the new null message */
if (oh->nmesgs >= oh->alloc_nmesgs)
if (H5O__alloc_msgs(oh, (size_t)1) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages");
/* Increment new gap size */
oh->chunk[chunkno].gap += new_gap_size;
/* Create new null message, with the tail of the previous null message */
null_msg = &(oh->mesg[oh->nmesgs++]);
null_msg->type = H5O_MSG_NULL;
null_msg->native = NULL;
null_msg->raw_size = new_gap_size - (size_t)H5O_SIZEOF_MSGHDR_OH(oh);
null_msg->raw = (oh->chunk[chunkno].image + oh->chunk[chunkno].size) -
(H5O_SIZEOF_CHKSUM_OH(oh) + null_msg->raw_size);
null_msg->chunkno = chunkno;
/* Zero out new null message's raw data */
if (null_msg->raw_size)
memset(null_msg->raw, 0, null_msg->raw_size);
/* Mark message as dirty */
null_msg->dirty = TRUE;
/* Reset size of gap in chunk */
oh->chunk[chunkno].gap = 0;
} /* end if */
else
oh->chunk[chunkno].gap = new_gap_size;
/* Mark the chunk as modified */
*chk_dirtied = TRUE;
} /* end if */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5O__add_gap() */
/*-------------------------------------------------------------------------
* Function: H5O__eliminate_gap
*
* Purpose: Eliminate a gap in a chunk with a null message.
*
* Note: Sometimes this happens as a result of converting an existing
* non-null message to a null message, so we zero out the gap
* here, even though it might already be zero (when we're adding
* a gap to a chunk with an existing null message). (Mostly,
* this just simplifies the code, esp. with the necessary chunk
* locking -QAK)
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O__eliminate_gap(H5O_t *oh, hbool_t *chk_dirtied, H5O_mesg_t *mesg, uint8_t *gap_loc, size_t gap_size)
{
uint8_t *move_start, *move_end; /* Pointers to area of messages to move */
hbool_t null_before_gap; /* Flag whether the null message is before the gap or not */
FUNC_ENTER_PACKAGE_NOERR
/* check args */
assert(oh);
assert(oh->version > H5O_VERSION_1);
assert(chk_dirtied);
assert(mesg);
assert(gap_loc);
assert(gap_size);
/* Check if the null message is before or after the gap produced */
null_before_gap = (hbool_t)(mesg->raw < gap_loc);
/* Set up information about region of messages to move */
if (null_before_gap) {
move_start = mesg->raw + mesg->raw_size;
move_end = gap_loc;
} /* end if */
else {
move_start = gap_loc + gap_size;
move_end = mesg->raw - H5O_SIZEOF_MSGHDR_OH(oh);
} /* end else */
/* Check for messages between null message and gap */
if (move_end > move_start) {
unsigned u; /* Local index variable */
/* Look for messages that need to move, to adjust raw pointers in chunk */
/* (this doesn't change the moved messages 'dirty' state) */
for (u = 0; u < oh->nmesgs; u++) {
uint8_t *msg_start; /* Start of encoded message in chunk */
msg_start = oh->mesg[u].raw - H5O_SIZEOF_MSGHDR_OH(oh);
if (oh->mesg[u].chunkno == mesg->chunkno && (msg_start >= move_start && msg_start < move_end)) {
/* Move message's raw pointer in appropriate direction */
if (null_before_gap)
oh->mesg[u].raw += gap_size;
else
oh->mesg[u].raw -= gap_size;
} /* end if */
} /* end for */
/* Slide raw message info in chunk image */
if (null_before_gap)
/* Slide messages down */
memmove(move_start + gap_size, move_start, (size_t)(move_end - move_start));
else {
/* Slide messages up */
memmove(move_start - gap_size, move_start, (size_t)(move_end - move_start));
/* Adjust start of null message */
mesg->raw -= gap_size;
} /* end else */
}
else if (move_end == move_start && !null_before_gap) {
/* Slide null message up */
memmove(move_start - gap_size, move_start, mesg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh));
/* Adjust start of null message */
mesg->raw -= gap_size;
} /* end if */
/* Zero out addition to null message */
memset(mesg->raw + mesg->raw_size, 0, gap_size);
/* Adjust size of null message */
mesg->raw_size += gap_size;
/* Set the gap size to zero for the chunk */
oh->chunk[mesg->chunkno].gap = 0;
/* Mark null message as dirty */
mesg->dirty = TRUE;
*chk_dirtied = TRUE;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* H5O__eliminate_gap() */
/*-------------------------------------------------------------------------
*
* Function: H5O__alloc_null
*
* Purpose: Allocate room for a new message from a null message
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O__alloc_null(H5F_t *f, H5O_t *oh, size_t null_idx, const H5O_msg_class_t *new_type, void *new_native,
size_t new_size)
{
H5O_chunk_proxy_t *chk_proxy = NULL; /* Chunk that message is in */
hbool_t chk_dirtied = FALSE; /* Flags for unprotecting chunk */
H5O_mesg_t *alloc_msg; /* Pointer to null message to allocate out of */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* check args */
assert(oh);
assert(new_type);
assert(new_size);
/* Point to null message to allocate out of */
alloc_msg = &oh->mesg[null_idx];
/* Protect chunk */
if (NULL == (chk_proxy = H5O__chunk_protect(f, oh, alloc_msg->chunkno)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk");
/* Check if there's a need to split the null message */
if (alloc_msg->raw_size > new_size) {
/* Check for producing a gap in the chunk */
if ((alloc_msg->raw_size - new_size) < (size_t)H5O_SIZEOF_MSGHDR_OH(oh)) {
size_t gap_size = alloc_msg->raw_size - new_size; /* Size of gap produced */
/* Adjust the size of the null message being eliminated */
alloc_msg->raw_size = new_size;
/* Add the gap to the chunk */
if (H5O__add_gap(f, oh, alloc_msg->chunkno, &chk_dirtied, null_idx,
alloc_msg->raw + alloc_msg->raw_size, gap_size) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't insert gap in chunk");
} /* end if */
else {
size_t new_mesg_size =
new_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh); /* Total size of newly allocated message */
H5O_mesg_t *null_msg; /* Pointer to new null message */
/* Check if we need to extend message table to hold the new null message */
if (oh->nmesgs >= oh->alloc_nmesgs) {
if (H5O__alloc_msgs(oh, (size_t)1) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages");
/* "Retarget" 'alloc_msg' pointer into newly re-allocated array of messages */
alloc_msg = &oh->mesg[null_idx];
} /* end if */
/* Create new null message, with the tail of the previous null message */
null_msg = &(oh->mesg[oh->nmesgs++]);
null_msg->type = H5O_MSG_NULL;
null_msg->native = NULL;
null_msg->raw = alloc_msg->raw + new_mesg_size;
null_msg->raw_size = alloc_msg->raw_size - new_mesg_size;
null_msg->chunkno = alloc_msg->chunkno;
/* Mark the message as dirty */
null_msg->dirty = TRUE;
chk_dirtied = TRUE;
/* Check for gap in new null message's chunk */
if (oh->chunk[null_msg->chunkno].gap > 0) {
unsigned null_chunkno = null_msg->chunkno; /* Chunk w/gap */
/* Eliminate the gap in the chunk */
if (H5O__eliminate_gap(oh, &chk_dirtied, null_msg,
((oh->chunk[null_chunkno].image + oh->chunk[null_chunkno].size) -
(H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[null_chunkno].gap)),
oh->chunk[null_chunkno].gap) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTREMOVE, FAIL, "can't eliminate gap in chunk");
} /* end if */
/* Set the size of the new "real" message */
alloc_msg->raw_size = new_size;
} /* end else */
} /* end if */
/* Initialize the new message */
alloc_msg->type = new_type;
alloc_msg->native = new_native;
/* Mark the new message as dirty */
alloc_msg->dirty = TRUE;
chk_dirtied = TRUE;
done:
/* Release chunk */
if (chk_proxy && H5O__chunk_unprotect(f, chk_proxy, chk_dirtied) < 0)
HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk");
FUNC_LEAVE_NOAPI(ret_value)
} /* H5O__alloc_null() */
/*-------------------------------------------------------------------------
*
* Function: H5O__alloc_msgs
*
* Purpose: Allocate more messages for a header
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
herr_t
H5O__alloc_msgs(H5O_t *oh, size_t min_alloc)
{
size_t old_alloc; /* Old number of messages allocated */
size_t na; /* New number of messages allocated */
H5O_mesg_t *new_mesg; /* Pointer to new message array */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* check args */
assert(oh);
/* Initialize number of messages information */
old_alloc = oh->alloc_nmesgs;
na = oh->alloc_nmesgs + MAX(oh->alloc_nmesgs, min_alloc); /* At least double */
/* Attempt to allocate more memory */
if (NULL == (new_mesg = H5FL_SEQ_REALLOC(H5O_mesg_t, oh->mesg, na)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed");
/* Update ohdr information */
oh->alloc_nmesgs = na;
oh->mesg = new_mesg;
/* Set new object header info to zeros */
memset(&oh->mesg[old_alloc], 0, (oh->alloc_nmesgs - old_alloc) * sizeof(H5O_mesg_t));
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5O__alloc_msgs() */
/*-------------------------------------------------------------------------
*
* Function: H5O__alloc_extend_chunk
*
* Purpose: Attempt to extend a chunk that is allocated on disk.
*
* If the extension is successful, and if the last message
* of the chunk is the null message, then that message will
* be extended with the chunk. Otherwise a new null message
* is created.
*
* f is the file in which the chunk will be written. It is
* included to ensure that there is enough space to extend
* this chunk.
*
* Return: TRUE: The chunk has been extended, and *msg_idx
* contains the message index for null message
* which is large enough to hold size bytes.
*
* FALSE: The chunk cannot be extended, and *msg_idx
* is undefined.
*
* FAIL: Some internal error has been detected.
*
*-------------------------------------------------------------------------
*/
static htri_t
H5O__alloc_extend_chunk(H5F_t *f, H5O_t *oh, unsigned chunkno, size_t size, size_t *msg_idx)
{
H5O_chunk_proxy_t *chk_proxy = NULL; /* Chunk that message is in */
hbool_t chk_dirtied = FALSE; /* Flag for unprotecting chunk */
size_t delta; /* Change in chunk's size */
size_t aligned_size = H5O_ALIGN_OH(oh, size);
uint8_t *old_image; /* Old address of chunk's image in memory */
size_t old_size; /* Old size of chunk */
htri_t was_extended; /* If chunk can be extended */
size_t extend_msg = 0; /* Index of null message to extend */
hbool_t extended_msg = FALSE; /* Whether an existing message was extended */
uint8_t new_size_flags = 0; /* New chunk #0 size flags */
hbool_t adjust_size_flags = FALSE; /* Whether to adjust the chunk #0 size flags */
size_t extra_prfx_size = 0; /* Extra bytes added to object header prefix */
size_t u; /* Local index variable */
htri_t ret_value = TRUE; /* return value */
FUNC_ENTER_PACKAGE
/* check args */
assert(f != NULL);
assert(oh != NULL);
assert(chunkno < oh->nchunks);
assert(size > 0);
assert(msg_idx != NULL);
assert(H5_addr_defined(oh->chunk[chunkno].addr));
/* Test to see if the specified chunk ends with a null messages.
* If successful, set the index of the null message in extend_msg.
*/
for (u = 0; u < oh->nmesgs; u++) {
/* Check for null message at end of proper chunk */
/* (account for possible checksum at end of chunk) */
if (oh->mesg[u].chunkno == chunkno && H5O_NULL_ID == oh->mesg[u].type->id &&
((oh->mesg[u].raw + oh->mesg[u].raw_size) ==
((oh->chunk[chunkno].image + oh->chunk[chunkno].size) -
(oh->chunk[chunkno].gap + H5O_SIZEOF_CHKSUM_OH(oh))))) {
extend_msg = u;
extended_msg = TRUE;
break;
} /* end if */
} /* end for */
/* If we can extend an existing null message, adjust the delta appropriately */
if (extended_msg) {
assert(oh->chunk[chunkno].gap == 0);
delta = aligned_size - oh->mesg[extend_msg].raw_size;
} /* end if */
else
delta = (aligned_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)) - oh->chunk[chunkno].gap;
delta = H5O_ALIGN_OH(oh, delta);
/* Check for changing the chunk #0 data size enough to need adjusting the flags */
if (oh->version > H5O_VERSION_1 && chunkno == 0) {
uint64_t chunk0_size; /* Size of chunk 0's data */
size_t orig_prfx_size = (size_t)1 << (oh->flags & H5O_HDR_CHUNK0_SIZE); /* Original prefix size */
assert(oh->chunk[0].size >= (size_t)H5O_SIZEOF_HDR(oh));
chunk0_size = oh->chunk[0].size - (size_t)H5O_SIZEOF_HDR(oh);
/* Check for moving to a 8-byte size encoding */
if (orig_prfx_size < 8 && (chunk0_size + delta) > 4294967295) {
extra_prfx_size = 8 - orig_prfx_size;
new_size_flags = H5O_HDR_CHUNK0_8;
adjust_size_flags = TRUE;
} /* end if */
/* Check for moving to a 4-byte size encoding */
else if (orig_prfx_size < 4 && (chunk0_size + delta) > 65535) {
extra_prfx_size = 4 - orig_prfx_size;
new_size_flags = H5O_HDR_CHUNK0_4;
adjust_size_flags = TRUE;
} /* end if */
/* Check for moving to a 2-byte size encoding */
else if (orig_prfx_size < 2 && (chunk0_size + delta) > 255) {
extra_prfx_size = 2 - orig_prfx_size;
new_size_flags = H5O_HDR_CHUNK0_2;
adjust_size_flags = TRUE;
} /* end if */
} /* end if */
/* Protect chunk */
if (NULL == (chk_proxy = H5O__chunk_protect(f, oh, chunkno)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk");
/* Determine whether the chunk can be extended */
was_extended = H5MF_try_extend(f, H5FD_MEM_OHDR, oh->chunk[chunkno].addr,
(hsize_t)(oh->chunk[chunkno].size), (hsize_t)(delta + extra_prfx_size));
if (was_extended < 0) /* error */
HGOTO_ERROR(H5E_OHDR, H5E_CANTEXTEND, FAIL, "can't tell if we can extend chunk");
else if (was_extended == FALSE) /* can't extend -- we are done */
HGOTO_DONE(FALSE);
/* Adjust object header prefix flags */
if (adjust_size_flags) {
oh->flags = (uint8_t)(oh->flags & ~H5O_HDR_CHUNK0_SIZE);
oh->flags |= new_size_flags;
/* Mark object header as dirty in cache */
if (H5AC_mark_entry_dirty(oh) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTMARKDIRTY, FAIL, "unable to mark object header as dirty");
} /* end if */
/* If we can extend an existing null message, take care of that */
if (extended_msg) {
/* Adjust message size of existing null message */
oh->mesg[extend_msg].raw_size += delta;
} /* end if */
/* Create new null message for end of chunk */
else {
/* Create a new null message */
if (oh->nmesgs >= oh->alloc_nmesgs)
if (H5O__alloc_msgs(oh, (size_t)1) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages");
/* Set extension message */
extend_msg = oh->nmesgs++;
/* Initialize new null message */
oh->mesg[extend_msg].type = H5O_MSG_NULL;
oh->mesg[extend_msg].native = NULL;
oh->mesg[extend_msg].raw = ((oh->chunk[chunkno].image + oh->chunk[chunkno].size) -
(H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[chunkno].gap)) +
H5O_SIZEOF_MSGHDR_OH(oh);
oh->mesg[extend_msg].raw_size = (delta + oh->chunk[chunkno].gap) - (size_t)H5O_SIZEOF_MSGHDR_OH(oh);
oh->mesg[extend_msg].chunkno = chunkno;
} /* end else */
/* Mark the extended message as dirty */
oh->mesg[extend_msg].dirty = TRUE;
chk_dirtied = TRUE;
/* Allocate more memory space for chunk's image */
old_image = oh->chunk[chunkno].image;
old_size = oh->chunk[chunkno].size;
oh->chunk[chunkno].size += delta + extra_prfx_size;
oh->chunk[chunkno].image = H5FL_BLK_REALLOC(chunk_image, old_image, oh->chunk[chunkno].size);
if (NULL == oh->chunk[chunkno].image)
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't reallocate extended object header chunk");
oh->chunk[chunkno].gap = 0;
/* Wipe new space for chunk */
memset(oh->chunk[chunkno].image + old_size, 0, oh->chunk[chunkno].size - old_size);
/* Move chunk 0 data up if the size flags changed */
if (adjust_size_flags)
memmove(oh->chunk[0].image + H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh),
oh->chunk[0].image + H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh) - extra_prfx_size,
old_size - (size_t)H5O_SIZEOF_HDR(oh) + extra_prfx_size);
/* Spin through existing messages, adjusting them */
for (u = 0; u < oh->nmesgs; u++) {
/* Adjust raw addresses for messages in this chunk to reflect new 'image' address */
if (oh->mesg[u].chunkno == chunkno)
oh->mesg[u].raw = oh->chunk[chunkno].image + extra_prfx_size + (oh->mesg[u].raw - old_image);
/* Find continuation message which points to this chunk and adjust chunk's size */
/* (Chunk 0 doesn't have a continuation message that points to it,
* its size is directly encoded in the object header) */
if (chunkno > 0 && (H5O_CONT_ID == oh->mesg[u].type->id) &&
(((H5O_cont_t *)(oh->mesg[u].native))->chunkno == chunkno)) {
H5O_chunk_proxy_t *chk_proxy2 = NULL; /* Chunk that continuation message is in */
hbool_t chk_dirtied2 = FALSE; /* Flag for unprotecting chunk */
unsigned cont_chunkno = oh->mesg[u].chunkno; /* Chunk # for continuation message */
/* Protect chunk containing continuation message */
if (NULL == (chk_proxy2 = H5O__chunk_protect(f, oh, cont_chunkno)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk");
/* Adjust size in continuation message */
assert(((H5O_cont_t *)(oh->mesg[u].native))->size == old_size);
((H5O_cont_t *)(oh->mesg[u].native))->size = oh->chunk[chunkno].size;
/* Flag continuation message as dirty */
oh->mesg[u].dirty = TRUE;
chk_dirtied2 = TRUE;
/* Release chunk containing continuation message */
if (H5O__chunk_unprotect(f, chk_proxy2, chk_dirtied2) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk");
} /* end if */
} /* end for */
/* Resize the chunk in the cache */
if (H5O__chunk_resize(oh, chk_proxy) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTRESIZE, FAIL, "unable to resize object header chunk");
/* Set new message index */
*msg_idx = extend_msg;
done:
/* Release chunk */
if (chk_proxy && H5O__chunk_unprotect(f, chk_proxy, chk_dirtied) < 0)
HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk");
FUNC_LEAVE_NOAPI(ret_value)
} /* H5O__alloc_extend_chunk() */
/*-------------------------------------------------------------------------
* Function: H5O__alloc_find_best_nonnull
*
* Purpose: Find the best fit non-null message for a given size of message
* to allocate.
*
* Note: The algorithm for finding a message to replace with a
* continuation message is still fairly limited. It's possible
* that two (or more) messages smaller than a continuation message
* might occupy a chunk and need to be moved in order to make
* room for the continuation message.
*
* Also, we aren't checking for NULL messages in front of another
* message right now...
*
* Return: Success: Index number of the null message for the
* new chunk. The null message will be at
* least SIZE bytes not counting the message
* ID or size fields.
*
* Failure: Negative
*
*-------------------------------------------------------------------------
*/
static herr_t
H5O__alloc_find_best_nonnull(const H5F_t *f, const H5O_t *oh, size_t *size, H5O_msg_alloc_info_t *found_msg)
{
H5O_mesg_t *curr_msg; /* Pointer to current message to operate on */
size_t cont_size; /* Continuation message size */
size_t multi_size; /* Size of all the messages in the last chunk */
unsigned u; /* Local index variable */
FUNC_ENTER_PACKAGE_NOERR
/* Check args */
assert(f);
assert(oh);
assert(size);
assert(*size > 0);
assert(found_msg);
/*
* Find the smallest message that could be moved to make room for the
* continuation message.
*
* Don't ever move continuation message from one chunk to another.
*
* Avoid moving attributes when possible to preserve their
* ordering (although ordering is *not* guaranteed!).
*
*/
cont_size = H5O_ALIGN_OH(oh, (size_t)(H5F_SIZEOF_ADDR(f) + H5F_SIZEOF_SIZE(f)));
multi_size = 0;
for (u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++) {
/* Don't consider continuation messages (for now) */
if (H5O_CONT_ID != curr_msg->type->id) {
unsigned msg_chunkno = curr_msg->chunkno; /* Chunk that the message is in */
uint8_t *end_chunk_data =
(oh->chunk[msg_chunkno].image + oh->chunk[msg_chunkno].size) -
(H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[msg_chunkno].gap); /* End of message data in chunk */
uint8_t *end_msg = curr_msg->raw + curr_msg->raw_size; /* End of current message */
size_t gap_size = 0; /* Size of gap after current message */
size_t null_size = 0; /* Size of NULL message after current message */
unsigned null_msgno = 0; /* Index of NULL message after current message */
size_t total_size; /* Total size of available space "around" current message */
/* Check if the message is the last one in the chunk */
if (end_msg == end_chunk_data)
gap_size = oh->chunk[msg_chunkno].gap;
else {
H5O_mesg_t *tmp_msg; /* Temp. pointer to message to operate on */
unsigned v; /* Local index variable */
/* Check for null message after this message, in same chunk */
for (v = 0, tmp_msg = &oh->mesg[0]; v < oh->nmesgs; v++, tmp_msg++) {
if (tmp_msg->type->id == H5O_NULL_ID &&
(tmp_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh)) == end_msg) {
null_msgno = v;
null_size = (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + tmp_msg->raw_size;
break;
} /* end if */
/* XXX: Should also check for NULL message in front of current message... */
} /* end for */
} /* end else */
/* Add up current message's total available space */
total_size = curr_msg->raw_size + gap_size + null_size;
/* Check if message is large enough to hold continuation info */
if (total_size >= cont_size) {
hbool_t better = FALSE; /* Whether the current message is better than a previous one */
/* Check for first message that can be moved */
if (found_msg->msgno < 0)
better = TRUE;
else {
/* Prioritize moving non-attributes above attributes */
/* (Even attributes with an otherwise better fit */
if (found_msg->id == H5O_ATTR_ID && curr_msg->type->id != H5O_ATTR_ID)
better = TRUE;
/* Either two attributes, or two non-attributes */
else {
/* Take a smaller one */
if (total_size < found_msg->total_size)
better = TRUE;
/* If they are the same size, choose the earliest one
* in the chunk array */
/* (Could also bias toward message earlier / later
* chunk in, but shouldn't be a big deal - QAK, 2016/10/21)
*/
else if (total_size == found_msg->total_size) {
if (msg_chunkno < found_msg->chunkno)
better = TRUE;
} /* end else-if */
} /* end else */
} /* end else */
/* If we found a better message, keep its info */
if (better) {
found_msg->msgno = (int)u;
found_msg->id = curr_msg->type->id;
found_msg->chunkno = msg_chunkno;
found_msg->gap_size = gap_size;
found_msg->null_size = null_size;
found_msg->total_size = total_size;
found_msg->null_msgno = null_msgno;
} /* end if */
} /* end if */
else if (found_msg->msgno < 0 && msg_chunkno == oh->nchunks - 1)
/* Keep track of the total size of smaller messages in the last
* chunk, in case we need to move more than 1 message.
*/
multi_size += curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh);
} /* end if */
} /* end for */
/*
* If we must move some other message to make room for the null
* message, then make sure the new chunk has enough room for that
* other message.
*
* Move other messages first, and attributes only as a last resort.
*
* If all else fails, move every message in the last chunk.
*
*/
if (found_msg->msgno < 0)
*size += multi_size;
else
*size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_msg->msgno].raw_size;
FUNC_LEAVE_NOAPI(SUCCEED)
} /* H5O__alloc_find_best_nonnull() */
/*-------------------------------------------------------------------------
* Function: H5O__alloc_chunk
*
* Purpose: Allocates and initializes a new chunk for the object header,
* including file space.
*
* Return: Success: SUCCEED, with chunk number for the
* new chunk and a pointer to the location in its
* image where the first message should be placed.
*
* Failure: Negative
*
*-------------------------------------------------------------------------
*/
herr_t
H5O__alloc_chunk(H5F_t *f, H5O_t *oh, size_t size, size_t found_null, const H5O_msg_alloc_info_t *found_msg,
size_t *new_idx)
{
H5O_mesg_t *curr_msg; /* Pointer to current message to operate on */
H5O_chunk_proxy_t *chk_proxy; /* Chunk that message is in */
size_t cont_size; /*continuation message size */
size_t idx; /* Message number */
uint8_t *p = NULL; /* Pointer into new chunk image */
H5O_cont_t *cont = NULL; /*native continuation message */
unsigned chunkno; /* Chunk allocated */
haddr_t new_chunk_addr; /* Address of new chunk in file */
unsigned u; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
/* check args */
assert(f);
assert(oh);
assert(found_msg);
assert(new_idx);
/*
* The total chunk size must include the requested space plus enough
* for the message header. This must be at least some minimum and
* aligned properly.
*/
size = MAX(H5O_MIN_SIZE, size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh));
assert(size == H5O_ALIGN_OH(oh, size));
/*
* The total chunk size must include enough space for the checksum
* on the chunk and the continuation chunk magic #. (which are only present
* in later versions of the object header)
*/
size += H5O_SIZEOF_CHKHDR_OH(oh);
/* Allocate space in file to hold the new chunk */
new_chunk_addr = H5MF_alloc(f, H5FD_MEM_OHDR, (hsize_t)size);
if (!H5_addr_defined(new_chunk_addr))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "unable to allocate space for new chunk");
/* Create the new chunk giving it a file address. */
if (oh->nchunks >= oh->alloc_nchunks) {
size_t na = MAX(H5O_NCHUNKS, oh->alloc_nchunks * 2); /* Double # of chunks allocated */
H5O_chunk_t *x;
if (NULL == (x = H5FL_SEQ_REALLOC(H5O_chunk_t, oh->chunk, na)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate larger chunk array, na = %zu", na);
oh->alloc_nchunks = na;
oh->chunk = x;
} /* end if */
H5_CHECKED_ASSIGN(chunkno, unsigned, oh->nchunks, size_t);
oh->nchunks++;
oh->chunk[chunkno].addr = new_chunk_addr;
oh->chunk[chunkno].size = size;
oh->chunk[chunkno].gap = 0;
if (NULL == (oh->chunk[chunkno].image = p = H5FL_BLK_CALLOC(chunk_image, size)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate image for chunk, size = %zu", size);
oh->chunk[chunkno].chunk_proxy = NULL;
/* If this is a later version of the object header format, put the magic
* # at the beginning of the chunk image.
*/
if (oh->version > H5O_VERSION_1) {
H5MM_memcpy(p, H5O_CHK_MAGIC, (size_t)H5_SIZEOF_MAGIC);
p += H5_SIZEOF_MAGIC;
} /* end if */
/*
* Make sure we have enough space for all possible new messages
* that could be generated below.
*/
if (oh->nmesgs + 3 > oh->alloc_nmesgs)
if (H5O__alloc_msgs(oh, (size_t)3) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages");
/* Check if we need to move multiple messages, in order to make room for the new message */
cont_size = H5O_ALIGN_OH(oh, (size_t)(H5F_SIZEOF_ADDR(f) + H5F_SIZEOF_SIZE(f)));
if (found_null >= oh->nmesgs) {
if (found_msg->msgno < 0) {
/* Move all non-null messages in the last chunk to the new chunk. This
* should be extremely rare so we don't care too much about minimizing
* the space used.
*/
H5O_mesg_t *null_msg; /* Pointer to new null message */
/* Protect last chunk */
if (NULL == (chk_proxy = H5O__chunk_protect(f, oh, chunkno - 1)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk");
/* Copy each message to the new location */
for (u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++)
if (curr_msg->chunkno == chunkno - 1) {
if (curr_msg->type->id == H5O_NULL_ID) {
/* Delete the null message */
if (u < oh->nmesgs - 1)
memmove(curr_msg, curr_msg + 1, ((oh->nmesgs - 1) - u) * sizeof(H5O_mesg_t));
oh->nmesgs--;
} /* end if */
else {
assert(curr_msg->type->id != H5O_CONT_ID);
/* Copy the raw data */
H5MM_memcpy(p, curr_msg->raw - (size_t)H5O_SIZEOF_MSGHDR_OH(oh),
curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh));
/* Update the message info */
curr_msg->chunkno = chunkno;
curr_msg->raw = p + H5O_SIZEOF_MSGHDR_OH(oh);
/* Account for copied message in new chunk */
p += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size;
size -= (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size;
} /* end else */
} /* end if */
/* Create a null message spanning the entire last chunk */
found_null = oh->nmesgs++;
null_msg = &(oh->mesg[found_null]);
null_msg->type = H5O_MSG_NULL;
null_msg->dirty = TRUE;
null_msg->native = NULL;
null_msg->raw = oh->chunk[chunkno - 1].image +
((chunkno == 1) ? H5O_SIZEOF_HDR(oh) : H5O_SIZEOF_CHKHDR_OH(oh)) -
H5O_SIZEOF_CHKSUM_OH(oh) + H5O_SIZEOF_MSGHDR_OH(oh);
null_msg->raw_size =
oh->chunk[chunkno - 1].size -
((chunkno == 1) ? (size_t)H5O_SIZEOF_HDR(oh) : (size_t)H5O_SIZEOF_CHKHDR_OH(oh)) -
(size_t)H5O_SIZEOF_MSGHDR_OH(oh);
null_msg->chunkno = chunkno - 1;
assert(null_msg->raw_size >= cont_size);
/* Remove any gap in the chunk */
oh->chunk[chunkno - 1].gap = 0;
/* Release chunk, marking it dirty */
if (H5O__chunk_unprotect(f, chk_proxy, TRUE) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk");
} /* end if */
else {
/* Move message (that will be replaced with continuation message)
* to new chunk, if necessary.
*/
H5O_mesg_t *null_msg; /* Pointer to new null message */
/* Protect chunk */
if (NULL == (chk_proxy = H5O__chunk_protect(f, oh, oh->mesg[found_msg->msgno].chunkno)))
HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk");
/* Create null message for space that message to copy currently occupies */
found_null = oh->nmesgs++;
null_msg = &(oh->mesg[found_null]);
null_msg->type = H5O_MSG_NULL;