diff --git a/src/gmt_api.c b/src/gmt_api.c
index df57d004060..9537dca4bd1 100644
--- a/src/gmt_api.c
+++ b/src/gmt_api.c
@@ -3632,9 +3632,11 @@ GMT_LOCAL struct GMT_DATASET * gmtapi_import_dataset (struct GMTAPI_CTRL *API, i
 	struct GMT_DATASEGMENT *S = NULL;
 	struct GMT_MATRIX *M_obj = NULL;
 	struct GMT_VECTOR *V_obj = NULL;
-	struct GMT_DATASET_HIDDEN *DH = NULL;
+	struct GMT_DATASET_HIDDEN *DH = NULL, *DHi = NULL;
 	struct GMT_DATATABLE_HIDDEN *TH = NULL;
 	struct GMT_DATASEGMENT_HIDDEN *SH = NULL;
+	struct GMT_MATRIX_HIDDEN *MH = NULL;
+	struct GMT_VECTOR_HIDDEN *VH = NULL;
 	struct GMTAPI_DATA_OBJECT *S_obj = NULL;
 	struct GMT_CTRL *GMT = API->GMT;
 
@@ -3757,19 +3759,25 @@ GMT_LOCAL struct GMT_DATASET * gmtapi_import_dataset (struct GMTAPI_CTRL *API, i
 				if (GMT->common.q.mode == GMT_RANGE_ROW_IN || GMT->common.q.mode == GMT_RANGE_DATA_IN)
 					GMT_Report (API, GMT_MSG_WARNING, "Row-selection via -qi is not implemented for GMT_IS_REFERENCE with GMT_IS_DATASET external memory objects\n");
 				GMT_Report (API, GMT_MSG_INFORMATION, "Referencing data table from GMT_DATASET memory location\n");
+				DHi = gmt_get_DD_hidden (Din_obj);
 				if ((tbl + Din_obj->n_tables) >= n_alloc) {	/* Need more space to hold these new tables */
 					n_alloc += Din_obj->n_tables;
 					D_obj->table = gmt_M_memory (GMT, D_obj->table, n_alloc, struct GMT_DATATABLE *);
 				}
-				for (tbl_in = 0; tbl_in < Din_obj->n_tables; tbl_in++, tbl++)	/* Pass over the pointers only */
+				for (tbl_in = 0; tbl_in < Din_obj->n_tables; tbl_in++, tbl++) {	/* Pass over the pointers only */
 					D_obj->table[tbl] = Din_obj->table[tbl_in];
+					Din_obj->table[tbl_in] = NULL;	/* Since passed to D_obj */
+				}
+				Din_obj->n_tables = 0;	/* Only the husk remains of this fruit */
 				D_obj->n_tables = tbl;
 				D_obj->geometry = S_obj->geometry;	/* Since provided when registered */
+				DH->alloc_mode = DHi->alloc_mode;	/* Must use whatever alloc_mode the input reference had */
+				DH->alloc_level = DHi->alloc_level;	/* Must use whatever alloc_level the input reference had */
 				check_col_switch = true;
 				update = regit = via = true;		/* Have reason to update min/max as well as registering D_obj when done */
 				break;
 
-		 	case GMT_IS_DUPLICATE|GMT_VIA_MATRIX:
+		 	case GMT_IS_DUPLICATE|GMT_VIA_MATRIX:	/* There is no difference since in both cases we must allocate dataset arrays */
 		 	case GMT_IS_REFERENCE|GMT_VIA_MATRIX:
 				/* Each matrix source becomes a separate table with a single segment unless there are NaN-records as segment headers */
 				if ((M_obj = S_obj->resource) == NULL) {
@@ -3916,8 +3924,10 @@ GMT_LOCAL struct GMT_DATASET * gmtapi_import_dataset (struct GMTAPI_CTRL *API, i
 					gmt_M_free (GMT, D_obj);	return_null (API, GMT_PTR_IS_NULL);
 				}
 				if (V_obj->type[0] != GMT_DOUBLE) {
+					GMT_Report (API, GMT_MSG_ERROR, "Only double-precision vectors can be passed via reference to datasets\n");
 					gmt_M_free (GMT, D_obj);	return_null (API, GMT_NOT_A_VALID_TYPE);
 				}
+				VH = gmt_get_V_hidden (V_obj);
 				if (GMT->common.q.mode == GMT_RANGE_ROW_IN || GMT->common.q.mode == GMT_RANGE_DATA_IN)
 					GMT_Report (API, GMT_MSG_WARNING, "Row-selection via -qi is not implemented for GMT_IS_REFERENCE|GMT_VIA_VECTOR external memory objects\n");
 				/* Each column double array source becomes preallocated column arrays in a separate table with a single segment */
@@ -3928,7 +3938,8 @@ GMT_LOCAL struct GMT_DATASET * gmtapi_import_dataset (struct GMTAPI_CTRL *API, i
 				            V_obj->n_columns, V_obj->n_rows);
 				D_obj->table[D_obj->n_tables] = gmt_get_table (GMT);
 				D_obj->table[D_obj->n_tables]->segment = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASEGMENT *);
-				D_obj->table[D_obj->n_tables]->segment[0] = GMT_Alloc_Segment (API, smode, 0, n_columns, NULL, NULL);
+				S = D_obj->table[D_obj->n_tables]->segment[0] = GMT_Alloc_Segment (API, smode, 0, n_columns, NULL, NULL);
+				SH = gmt_get_DS_hidden (S);
 				for (col = 0; col < V_obj->n_columns; col++) {
 					if (GMT->common.i.select)	/* -i has selected some columns */
 						col_pos = GMT->current.io.col[GMT_IN][col].col;	/* Which data column to pick */
@@ -3936,10 +3947,11 @@ GMT_LOCAL struct GMT_DATASET * gmtapi_import_dataset (struct GMTAPI_CTRL *API, i
 						col_pos = 1 - col;	/* Read lat/lon instead of lon/lat */
 					else
 						col_pos = col;	/* Just goto that column */
-					D_obj->table[D_obj->n_tables]->segment[0]->data[col] = V_obj->data[col_pos].f8;
+					S->data[col] = V_obj->data[col_pos].f8;
+					SH->alloc_mode[col] = VH->alloc_mode[col];	/* Inherit from what we got */
 				}
 				DH = gmt_get_DD_hidden (D_obj);
-				if (smode) D_obj->table[D_obj->n_tables]->segment[0]->text = V_obj->text;
+				if (smode) S->text = V_obj->text;
 				gmtapi_increment_d (D_obj, V_obj->n_rows, n_columns, 1U);	/* Update counters for D_obj with 1 segment */
 				DH->alloc_mode = GMT_ALLOC_EXTERNALLY;	/* Since we just hooked on the arrays */
 				new_ID = GMT_Register_IO (API, GMT_IS_DATASET, GMT_IS_REFERENCE, geometry, GMT_IN, NULL, D_obj);	/* Register a new resource to hold D_obj */
@@ -3994,6 +4006,8 @@ GMT_LOCAL struct GMT_DATASET * gmtapi_import_dataset (struct GMTAPI_CTRL *API, i
 			return_null (API, GMT_OBJECT_NOT_FOUND);	/* Some internal error... */
 		API->object[new_item]->resource = D_obj;
 		API->object[new_item]->status = GMT_IS_USED;	/* Mark as read */
+		API->object[new_item]->alloc_mode = DH->alloc_mode;	/* Clarify allocation mode for this object */
+		API->object[new_item]->alloc_level = DH->alloc_level;
 	}
 	if (D_obj->n_tables == 0) {	/* Only found empty files (e.g., /dev/null) and we have nothing to show for our efforts.  Return an single empty table with no segments. */
 		D_obj->table = gmt_M_memory (GMT, D_obj->table, 1, struct GMT_DATATABLE *);
@@ -4321,6 +4335,7 @@ GMT_LOCAL int gmtapi_export_dataset (struct GMTAPI_CTRL *API, int object_ID, uns
 					V_obj->type[col] = S_obj->type;	/* Set same data type for all columns */
 					V_obj->data[col].f8 = S->data[col];	/* Set pointer only */
 					VH->alloc_mode[col] = GMT_ALLOC_EXTERNALLY;	/* Since not duplicated, just pointed to */
+					SH->alloc_mode[col] = GMT_ALLOC_EXTERNALLY;	/* To prevent freeing in D_obj */
 				}
 				if (S->text) {
 					V_obj->text = S->text;
@@ -4329,7 +4344,6 @@ GMT_LOCAL int gmtapi_export_dataset (struct GMTAPI_CTRL *API, int object_ID, uns
 				V_obj->n_rows = n_rows;
 				VH->alloc_level = S_obj->alloc_level;	/* Otherwise D_obj will be freed before we get to use data */
 				S_obj->alloc_mode = DH->alloc_mode;	/* Otherwise D_obj will be freed before we get to use data */
-				SH->alloc_mode = GMT_ALLOC_EXTERNALLY;	/* To prevent freeing in D_obj */
 			}
 			else {	/* Got a preallocated container */
 				if (V_obj->n_rows < n_rows || V_obj->n_columns < n_columns)
@@ -8633,7 +8647,7 @@ GMT_LOCAL int gmtapi_end_io_dataset (struct GMTAPI_CTRL *API, struct GMTAPI_DATA
 	DH = gmt_get_DD_hidden (D);
 	TH = gmt_get_DT_hidden (T);
 	if (count[GMT_SEG] >= 0) {	/* Finalize segment allocations */
-		if (!T->segment[count[GMT_SEG]]) T->segment[count[GMT_SEG]] = gmt_get_segment (API->GMT);
+		if (!T->segment[count[GMT_SEG]]) T->segment[count[GMT_SEG]] = gmt_get_segment (API->GMT, T->n_columns);
 		gmtlib_assign_segment (API->GMT, GMT_OUT, T->segment[count[GMT_SEG]], count[GMT_ROW], T->n_columns);	/* Allocate and place arrays into segment */
 		count[GMT_SEG]++;	/* Set final number of segments */
 		T->n_segments++;
@@ -10143,7 +10157,7 @@ GMT_LOCAL int gmtapi_put_record_dataset (struct GMTAPI_CTRL *API, unsigned int m
 		case GMT_WRITE_SEGMENT_HEADER:	/* Export a segment header record; write NaNs if binary  */
 			count[GMT_SEG]++;	/* Start of new segment */
 			if (count[GMT_SEG]) {	/* Must first copy over records for the previous segments; last (or only) segment will be done by GMT_End_IO */
-				if (!T->segment[count[GMT_SEG]-1]) T->segment[count[GMT_SEG]-1] = gmt_get_segment (API->GMT);
+				if (!T->segment[count[GMT_SEG]-1]) T->segment[count[GMT_SEG]-1] = gmt_get_segment (API->GMT, T->n_columns);
 				gmtlib_assign_segment (GMT, GMT_OUT, T->segment[count[GMT_SEG]-1], count[GMT_ROW], T->n_columns);	/* Allocate and place arrays into previous segment */
 				count[GMT_ROW] = 0;	/* Reset for next segment */
 				T->n_segments++;
@@ -10153,7 +10167,7 @@ GMT_LOCAL int gmtapi_put_record_dataset (struct GMTAPI_CTRL *API, unsigned int m
 				T->segment = gmt_M_malloc (GMT, T->segment, count[GMT_SEG], &TH->n_alloc, struct GMT_DATASEGMENT *);
 				gmt_M_memset (&T->segment[was], TH->n_alloc - was, struct GMT_DATASEGMENT *);
 			}
-			if (!T->segment[count[GMT_SEG]]) T->segment[count[GMT_SEG]] = gmt_get_segment (GMT);
+			if (!T->segment[count[GMT_SEG]]) T->segment[count[GMT_SEG]] = gmt_get_segment (GMT, T->n_columns);
 			s = (record) ? (char *)record : GMT->current.io.segment_header;	/* Default to last segment header record if NULL */
 			if (s && strlen(s)) {	/* Found a segment header */
 				if (T->segment[count[GMT_SEG]]->header) gmt_M_str_free (T->segment[count[GMT_SEG]]->header);	/* Hm, better free the old guy before strdup'ing a new one */
@@ -14718,7 +14732,7 @@ void * GMT_Alloc_Segment (void *V_API, unsigned int mode, uint64_t n_rows, uint6
 	API->error = GMT_NOERROR;
 	if ((Snew = S) != NULL)	/* Existing segment given */
 		first = false;
-	else if ((Snew = gmt_get_segment (API->GMT)) == NULL) /* Something went wrong */
+	else if ((Snew = gmt_get_segment (API->GMT, n_columns)) == NULL) /* Something went wrong */
 		return_null (V_API, GMT_MEMORY_ERROR);
 		/* Only reallocate if desired n_rows differ from current n_rows */
 	alloc = (first || (n_rows && n_rows != Snew->n_rows));	/* Alloc first time or reallocate later if necessary */
diff --git a/src/gmt_dcw.c b/src/gmt_dcw.c
index f0bc2f553ec..3ed483924d4 100644
--- a/src/gmt_dcw.c
+++ b/src/gmt_dcw.c
@@ -631,7 +631,7 @@ struct GMT_DATASET * gmt_DCW_operation (struct GMT_CTRL *GMT, struct GMT_DCW_SEL
 				gmt_M_malloc2 (GMT, S->data[GMT_X], S->data[GMT_Y], S->n_rows, NULL, double);
 				gmt_M_memcpy (S->data[GMT_X], P->data[GMT_X], S->n_rows, double);
 				gmt_M_memcpy (S->data[GMT_Y], P->data[GMT_Y], S->n_rows, double);
-				SH->alloc_mode = GMT_ALLOC_INTERNALLY;	/* Allocated in GMT */
+				SH->alloc_mode[GMT_X] = SH->alloc_mode[GMT_Y] = GMT_ALLOC_INTERNALLY;	/* Allocated in GMT */
 				seg++;
 			}
 		}
diff --git a/src/gmt_hidden.h b/src/gmt_hidden.h
index 10cddefcc29..cf5cb92f893 100644
--- a/src/gmt_hidden.h
+++ b/src/gmt_hidden.h
@@ -79,7 +79,7 @@ struct GMT_DATASEGMENT_HIDDEN {    /* Supporting information hidden from the API
 	double lat_limit;		/* For polar caps: the latitude of the point closest to the pole */
 	struct GMT_OGR_SEG *ogr;	/* NULL unless OGR/GMT metadata exist for this segment */
 	struct GMT_DATASEGMENT *next;	/* NULL unless polygon and has holes and pointing to next hole */
-	enum GMT_enum_alloc alloc_mode;	/* Allocation mode [GMT_ALLOC_INTERNALLY] */
+	enum GMT_enum_alloc *alloc_mode;	/* Allocation mode per column [GMT_ALLOC_INTERNALLY] */
 	char *file[2];			/* Name of file or source [0 = in, 1 = out] */
 };
 
diff --git a/src/gmt_io.c b/src/gmt_io.c
index fd1a61d71b5..e6751b173bb 100644
--- a/src/gmt_io.c
+++ b/src/gmt_io.c
@@ -2936,12 +2936,14 @@ GMT_LOCAL int gmtio_prep_ogr_output (struct GMT_CTRL *GMT, struct GMT_DATASET *D
 		/* OK, successfully passed the constant column tests, if any */
 		for (seg = col = 0; seg < T->n_segments; seg++) {	/* Free up columns now stored as aspatial values */
 			S = T->segment[seg];
+			SH = gmt_get_DS_hidden (S);
 			for (k = 0; k < TH->ogr->n_aspatial; k++)
 				if (GMT->common.a.col[k] > 0)
 					gmt_M_free (GMT, S->data[GMT->common.a.col[k]]);
 			for (k = col = 0; k < T->n_columns; k++) {
 				while (!S->data[k]) k++;	/* Next available column */
-				S->data[col++] = S->data[k];	/* Update pointers */
+				S->data[col] = S->data[k];	/* Update pointers */
+				SH->alloc_mode[col++] = SH->alloc_mode[k];	/* Update modes */
 			}
 			S->n_columns = col;	/* May have lost some columns now */
 		}
@@ -3022,14 +3024,17 @@ GMT_LOCAL void gmtio_write_multilines (struct GMT_CTRL *GMT, FILE *fp, char *tex
 GMT_LOCAL void gmtio_adjust_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT *S, uint64_t n_columns) {
 	/* Change the number of columns in this segment to n_columns (free or allocate as needed) */
 	uint64_t col;
+	struct GMT_DATASEGMENT_HIDDEN *SH = gmt_get_DS_hidden (S);
 	for (col = n_columns; col < S->n_columns; col++) gmt_M_free (GMT, S->data[col]);	/* Free up if n_columns < S->columns */
 	S->data = gmt_M_memory (GMT, S->data, n_columns, double *);
 	S->min = gmt_M_memory (GMT, S->min, n_columns, double);
 	S->max = gmt_M_memory (GMT, S->max, n_columns, double);
+	SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, n_columns, enum GMT_enum_alloc);
 	for (col = S->n_columns; col < n_columns; col++) {	/* Allocate new columns and initialize the min/max arrays */
 		S->min[col] = +DBL_MAX;
 		S->max[col] = -DBL_MAX;
 		S->data[col] = gmt_M_memory (GMT, NULL, S->n_rows, double);
+		SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
 	}
 	S->n_columns = n_columns;
 }
@@ -7638,15 +7643,17 @@ int gmt_alloc_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT *S, uint64_t
 	 * If first is true then the segment arrays for data/text are allocated and min/max arrays set
 	 */
 	uint64_t col;
+	struct GMT_DATASEGMENT_HIDDEN *SH = gmt_get_DS_hidden (S);
 	if (first && n_columns) {	/* First time we allocate the number of columns needed */
-		S->data = gmt_M_memory (GMT, NULL, n_columns, double *);
-		S->min = gmt_M_memory (GMT, NULL, n_columns, double);
-		S->max = gmt_M_memory (GMT, NULL, n_columns, double);
+		S->data = gmt_M_memory (GMT, S->data, n_columns, double *);
+		S->min = gmt_M_memory (GMT, S->min, n_columns, double);
+		S->max = gmt_M_memory (GMT, S->max, n_columns, double);
 		for (col = 0; col < n_columns; col++) {	/* Initialize the min/max array */
 			S->min[col] = +DBL_MAX;
 			S->max[col] = -DBL_MAX;
 		}
 		S->n_columns = n_columns;
+		SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, n_columns, enum GMT_enum_alloc);
 	}
 	if (!first && S->n_columns != n_columns) {	/* Error */
 		GMT_Report (GMT->parent, GMT_MSG_ERROR, "gmt_alloc_segment: Cannot reallocate the number of columns in an existing segment");
@@ -7654,12 +7661,12 @@ int gmt_alloc_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT *S, uint64_t
 	}
 	S->n_rows  = n_rows;
 	if (n_rows) {	/* Allocate or reallocate column arrays */
-		struct GMT_DATASEGMENT_HIDDEN *SH = gmt_get_DS_hidden (S);
 		for (col = 0; col < n_columns; col++) {
 			if ((S->data[col] = gmt_M_memory (GMT, S->data[col], n_rows, double)) == NULL) {
 				GMT_Report (GMT->parent, GMT_MSG_ERROR, "gmt_alloc_segment: Unable to reallocate data column %" PRIu64 " to new length %" PRIu64 "\n", col, n_rows);
 				return 1;
 			}
+			SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
 		}
 		if (mode & GMT_WITH_STRINGS) {	/* Also allocate the string pointers */
 			if ((S->text = gmt_M_memory (GMT, S->text, n_rows, char *)) == NULL) {
@@ -7667,7 +7674,6 @@ int gmt_alloc_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT *S, uint64_t
 				return 1;
 			}
 		}
-		SH->alloc_mode = GMT_ALLOC_INTERNALLY;
 		SH->n_alloc = n_rows;
 	}
 	return (GMT_OK);
@@ -7682,9 +7688,10 @@ void gmtlib_assign_segment (struct GMT_CTRL *GMT, unsigned int direction, struct
 	struct GMT_DATASEGMENT_HIDDEN *SH = gmt_get_DS_hidden (S);
 	if (n_rows == 0) return;	/* Nothing to do */
 	/* First allocate struct member arrays */
-	S->data = gmt_M_memory (GMT, NULL, n_columns, double *);
-	S->min  = gmt_M_memory (GMT, NULL, n_columns, double);
-	S->max  = gmt_M_memory (GMT, NULL, n_columns, double);
+	S->data = gmt_M_memory (GMT, S->data, n_columns, double *);
+	S->min  = gmt_M_memory (GMT, S->min, n_columns, double);
+	S->max  = gmt_M_memory (GMT, S->max, n_columns, double);
+	SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, n_columns, enum GMT_enum_alloc);
 
 	if (n_rows > GMT_INITIAL_MEM_ROW_ALLOC) {	/* Large segment, just pass allocated pointers and start over with new tmp vectors later */
 		GMT_Report (GMT->parent, GMT_MSG_DEBUG, "gmtlib_assign_segment: Pass %" PRIu64 " large arrays with length = %"
@@ -7693,6 +7700,7 @@ void gmtlib_assign_segment (struct GMT_CTRL *GMT, unsigned int direction, struct
 			if (n_rows < GMT->hidden.mem_rows)
 				GMT->hidden.mem_coord[col] = gmt_M_memory (GMT, GMT->hidden.mem_coord[col], n_rows, double);	/* Trim back */
 			S->data[col] = GMT->hidden.mem_coord[col];	/* Pass the pointer */
+			SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
 			GMT->hidden.mem_coord[col] = NULL;		/* Null this out to start over for next segment */
 		}
 		if (GMT->current.io.record_type[direction] & GMT_READ_TEXT) {
@@ -7707,6 +7715,7 @@ void gmtlib_assign_segment (struct GMT_CTRL *GMT, unsigned int direction, struct
 		for (col = 0; col < n_columns; col++) {	/* Initialize the min/max array */
 			S->data[col] = gmt_M_memory (GMT, S->data[col], n_rows, double);
 			gmt_M_memcpy (S->data[col], GMT->hidden.mem_coord[col], n_rows, double);
+			SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
 		}
 		if (GMT->current.io.record_type[direction] & GMT_READ_TEXT) {
 			uint64_t row;
@@ -7719,7 +7728,6 @@ void gmtlib_assign_segment (struct GMT_CTRL *GMT, unsigned int direction, struct
 	}
 	S->n_rows = n_rows;
 	S->n_columns = n_columns;
-	SH->alloc_mode = GMT_ALLOC_INTERNALLY;
 }
 
 /*! . */
@@ -7746,22 +7754,34 @@ double *gmtlib_assign_vector (struct GMT_CTRL *GMT, uint64_t n_rows, uint64_t co
 
 struct GMT_DATASET * gmt_get_dataset (struct GMT_CTRL *GMT) {
 	struct GMT_DATASET *D = NULL;
+	struct GMT_DATASET_HIDDEN *DH = NULL;
 	D = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASET);
-	D->hidden = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASET_HIDDEN);
+	D->hidden = DH = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASET_HIDDEN);
+	DH->alloc_mode = GMT_ALLOC_INTERNALLY;	/* So GMT_* modules can free this memory */
+	DH->alloc_level = GMT->hidden.func_level;	/* Must be freed at this level */
 	return (D);
 }
 
 struct GMT_DATATABLE * gmt_get_table (struct GMT_CTRL *GMT) {
 	struct GMT_DATATABLE *T = NULL;
+	struct GMT_DATATABLE_HIDDEN *TH = NULL;
 	T = gmt_M_memory (GMT, NULL, 1, struct GMT_DATATABLE);
-	T->hidden = gmt_M_memory (GMT, NULL, 1, struct GMT_DATATABLE_HIDDEN);
+	T->hidden = TH = gmt_M_memory (GMT, NULL, 1, struct GMT_DATATABLE_HIDDEN);
+	TH->alloc_mode = GMT_ALLOC_INTERNALLY;	/* So GMT_* modules can free this memory */
+	TH->alloc_level = GMT->hidden.func_level;	/* Must be freed at this level */
 	return (T);
 }
 
-struct GMT_DATASEGMENT * gmt_get_segment (struct GMT_CTRL *GMT) {
+struct GMT_DATASEGMENT * gmt_get_segment (struct GMT_CTRL *GMT, uint64_t n_columns) {
+	uint64_t col;
 	struct GMT_DATASEGMENT *S = NULL;
+	struct GMT_DATASEGMENT_HIDDEN *SH = NULL;
 	S = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASEGMENT);
-	S->hidden = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASEGMENT_HIDDEN);
+	S->hidden = SH = gmt_M_memory (GMT, NULL, 1, struct GMT_DATASEGMENT_HIDDEN);
+	if (n_columns)
+		SH->alloc_mode = gmt_M_memory (GMT, NULL, n_columns, enum GMT_enum_alloc);
+	for (col = 0; col < n_columns; col++)
+		SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;	/* So GMT_* modules can free this memory */
 	return (S);
 }
 
@@ -7790,7 +7810,7 @@ struct GMT_DATATABLE * gmt_create_table (struct GMT_CTRL *GMT, uint64_t n_segmen
 	if (n_segments) {
 		T->segment = gmt_M_memory (GMT, NULL, n_segments, struct GMT_DATASEGMENT *);
 		for (seg = 0; alloc && seg < n_segments; seg++) {
-			T->segment[seg] = gmt_get_segment (GMT);
+			T->segment[seg] = gmt_get_segment (GMT, n_columns);
 			gmt_alloc_segment (GMT, T->segment[seg], n_rows, n_columns, mode, true);
 			if (alloc_only) T->segment[seg]->n_rows = 0;
 		}
@@ -7825,8 +7845,6 @@ struct GMT_DATASET * gmtlib_create_dataset (struct GMT_CTRL *GMT, uint64_t n_tab
 	for (tbl = 0; tbl < n_tables; tbl++)
 		if ((D->table[tbl] = gmt_create_table (GMT, n_segments, n_rows, n_columns, mode, alloc_only)) == NULL)
 			return (NULL);
-	DH->alloc_level = GMT->hidden.func_level;	/* Must be freed at this level. */
-	DH->alloc_mode = GMT_ALLOC_INTERNALLY;		/* So GMT_* modules can free this memory. */
 	DH->id = GMT->parent->unique_var_ID++;		/* Give unique identifier */
 
 	return (D);
@@ -7989,7 +8007,7 @@ struct GMT_DATATABLE * gmtlib_read_table (struct GMT_CTRL *GMT, void *source, un
 			/* To use different line-distances for each segment, place the distance in the segment header */
 			if (first_seg || T->segment[seg]->n_rows > 0) {
 				if (!first_seg) seg++;	/* Only advance segment if last had any points */
-				T->segment[seg] = gmt_get_segment (GMT);
+				T->segment[seg] = gmt_get_segment (GMT, T->n_columns);
 				first_seg = false;
 			}
 			n_read++;
@@ -8173,6 +8191,7 @@ unsigned int gmt_realloc_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D, u
 	unsigned int mode = 0;
 	struct GMT_DATATABLE *T = NULL;
 	struct GMT_DATASEGMENT *S;
+	struct GMT_DATASEGMENT_HIDDEN *SH = NULL;
 
 	if (D == NULL) return GMT_NOERROR;
 	if (D->table[0]->segment[0]->text) mode = GMT_WITH_STRINGS;
@@ -8180,7 +8199,7 @@ unsigned int gmt_realloc_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D, u
 	if (dim[GMT_TBL] && dim[GMT_TBL] < D->n_tables) {	/* Remove unneeded tables and reallocate array */
 		for (tbl = dim[GMT_TBL]; tbl < D->n_tables; tbl++)
 			gmt_free_table (GMT, D->table[tbl]);
-		gmt_M_memory (GMT, D->table, dim[GMT_TBL], struct GMT_DATATABLE *);
+		D->table = gmt_M_memory (GMT, D->table, dim[GMT_TBL], struct GMT_DATATABLE *);
 		D->n_tables = dim[GMT_TBL];
 	}
 	/* Here, any tables not needed have been removed */
@@ -8191,16 +8210,16 @@ unsigned int gmt_realloc_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D, u
 		if (dim[GMT_SEG] && dim[GMT_SEG] < T->n_segments) {	/* Remove unneeded segments and reallocate array for this table */
 			for (seg = dim[GMT_SEG]; seg < T->n_segments; seg++)
 				gmt_free_segment (GMT, &(T->segment[seg]));
-			gmt_M_memory (GMT, T->segment, dim[GMT_SEG], struct GMT_DATASEGMENT *);
+			T->segment = gmt_M_memory (GMT, T->segment, dim[GMT_SEG], struct GMT_DATASEGMENT *);
 			T->n_segments = dim[GMT_SEG];
 			/* This table now has max dim[SEG] segments but might have fewer */
 		}
 		else if (dim[GMT_SEG] > T->n_segments) {	/* Want to add more segments */
-			gmt_M_memory (GMT, T->segment, dim[GMT_SEG], struct GMT_DATASEGMENT *);
+			T->segment = gmt_M_memory (GMT, T->segment, dim[GMT_SEG], struct GMT_DATASEGMENT *);
 			for (seg = T->n_segments; seg < dim[GMT_SEG]; seg++) {
 				S = T->segment[seg];	/* Short-cut */
 				n_row = (dim[GMT_ROW] == 0) ? S->n_rows : dim[GMT_ROW];
-				T->segment[seg] = gmt_get_segment (GMT);
+				T->segment[seg] = gmt_get_segment (GMT, n_col);
 				gmt_alloc_segment (GMT, T->segment[seg], n_row, n_col, mode, false);
 				S->n_rows = n_row;
 			}
@@ -8208,35 +8227,40 @@ unsigned int gmt_realloc_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D, u
 
 		for (seg = 0; seg < T->n_segments; seg++) {	/* Check each of the segments of this table */
 			S = T->segment[seg];	/* Short-cut */
+			SH = gmt_get_DS_hidden (S);
 			n_row = (dim[GMT_ROW] == 0) ? S->n_rows : dim[GMT_ROW];
 			if (dim[GMT_COL] && dim[GMT_COL] < T->n_columns) {	/* Remove unneeded columns and reallocate array of columns */
 				for (col = dim[GMT_COL]; col < T->n_columns; col++)
 					gmt_M_free (GMT, S->data[col]);
 				S->n_columns = dim[GMT_COL];
-				gmt_M_memory (GMT, S->data, dim[GMT_COL], double *);
-				gmt_M_memory (GMT, S->min,  dim[GMT_COL], double *);
-				gmt_M_memory (GMT, S->max,  dim[GMT_COL], double *);
+				S->data = gmt_M_memory (GMT, S->data, dim[GMT_COL], double *);
+				S->min = gmt_M_memory (GMT, S->min, dim[GMT_COL], double *);
+				S->max = gmt_M_memory (GMT, S->max, dim[GMT_COL], double *);
+				SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, dim[GMT_COL], enum GMT_enum_alloc);
 				if (seg == 0) {	/* Only do this once for the table */
-					gmt_M_memory (GMT, T->min, dim[GMT_COL], double *);
-					gmt_M_memory (GMT, T->max, dim[GMT_COL], double *);
+					T->min = gmt_M_memory (GMT, T->min, dim[GMT_COL], double *);
+					T->max = gmt_M_memory (GMT, T->max, dim[GMT_COL], double *);
 					if (tbl == 0) {/* Only do this once for the dataset */
-						gmt_M_memory (GMT, D->min, dim[GMT_COL], double *);
-						gmt_M_memory (GMT, D->max, dim[GMT_COL], double *);
+						D->min = gmt_M_memory (GMT, D->min, dim[GMT_COL], double *);
+						D->max = gmt_M_memory (GMT, D->max, dim[GMT_COL], double *);
 					}
 				}
 			}
 			else if (dim[GMT_COL] > T->n_columns) {	/* Add new columns and reallocate arrays */
-				gmt_M_memory (GMT, S->data, dim[GMT_COL], double *);
-				gmt_M_memory (GMT, S->min,  dim[GMT_COL], double *);
-				gmt_M_memory (GMT, S->max,  dim[GMT_COL], double *);
-				for (col = T->n_columns; col < dim[GMT_COL]; col++)	/* Allocate the new data arrays */
-					gmt_M_memory (GMT, S->data[col], n_row, double);
+				S->data = gmt_M_memory (GMT, S->data, dim[GMT_COL], double *);
+				S->min = gmt_M_memory (GMT, S->min, dim[GMT_COL], double *);
+				S->max = gmt_M_memory (GMT, S->max, dim[GMT_COL], double *);
+				SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, dim[GMT_COL], enum GMT_enum_alloc);
+				for (col = T->n_columns; col < dim[GMT_COL]; col++) {	/* Allocate the new data arrays */
+					S->data[col] = gmt_M_memory (GMT, S->data[col], n_row, double);
+					SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
+				}
 			}
 			if (dim[GMT_ROW] && dim[GMT_ROW] < S->n_rows) {	/* Remove unneeded rows by reallocating arrays */
 				for (col = dim[GMT_COL]; col < S->n_columns; col++)
-					gmt_M_memory (GMT, S->data[col], dim[GMT_ROW], double);
+					S->data[col] = gmt_M_memory (GMT, S->data[col], dim[GMT_ROW], double);
 				if (S->text)
-					gmt_M_memory (GMT, S->text, dim[GMT_ROW], char *);
+					S->text = gmt_M_memory (GMT, S->text, dim[GMT_ROW], char *);
 				S->n_rows = dim[GMT_ROW];
 			}
 		}
@@ -8245,7 +8269,7 @@ unsigned int gmt_realloc_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D, u
 
 	/* 2. Now we can expand anything that should grow */
 	if (dim[GMT_TBL] > D->n_tables) {	/* Add new tables and reallocate array */
-		gmt_M_memory (GMT, D->table, dim[GMT_TBL], struct GMT_DATATABLE *);
+		D->table = gmt_M_memory (GMT, D->table, dim[GMT_TBL], struct GMT_DATATABLE *);
 		for (tbl = D->n_tables; tbl < dim[GMT_TBL]; tbl++) {
 			if ((D->table[tbl] = gmt_create_table (GMT, n_seg, n_row, n_col, mode, false)) == NULL)
 				return (GMT_MEMORY_ERROR);
@@ -8411,6 +8435,8 @@ void gmtlib_change_out_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D) {
 	double **temp = NULL;
 	struct GMT_DATATABLE *T = NULL;
 	struct GMT_DATASEGMENT *S;
+	struct GMT_DATASEGMENT_HIDDEN *SH = NULL;
+
 	if (!GMT->common.o.select) return;	/* Nutin' to do */
 	temp = gmt_M_memory (GMT, NULL, D->n_columns, double *);
 	used = gmt_M_memory (GMT, NULL, D->n_columns, unsigned int);
@@ -8422,10 +8448,14 @@ void gmtlib_change_out_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D) {
 		T = D->table[tbl];	/* Current atble */
 		for (seg = 0; seg < T->n_segments; seg++) {
 			S = T->segment[seg];	/* Current segment */
-			if (extend)	{/* Allocate more arrays first */
+			SH = gmt_get_DS_hidden (S);
+			if (extend)	{	/* Allocate more arrays first */
 				S->data = gmt_M_memory (GMT, S->data, GMT->common.o.n_cols, double *);
-				for (col = D->n_columns; col < GMT->common.o.n_cols; col++)
+				SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, GMT->common.o.n_cols, enum GMT_enum_alloc);
+				for (col = D->n_columns; col < GMT->common.o.n_cols; col++) {
 					S->data[col] = gmt_M_memory (GMT, NULL, S->n_rows, double);
+					SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
+				}
 			}
 			for (col = 0; col < S->n_columns; col++)	/* Keep pointers to original arrays */
 				temp[col] = S->data[col];
@@ -8451,7 +8481,10 @@ void gmtlib_change_out_dataset (struct GMT_CTRL *GMT, struct GMT_DATASET *D) {
 			if (adjust) {	/* Modify length of min/max arrays */
 				S->min  = gmt_M_memory (GMT, S->min, GMT->common.o.n_cols, double);
 				S->max  = gmt_M_memory (GMT, S->max, GMT->common.o.n_cols, double);
-				if (!extend) S->data = gmt_M_memory (GMT, S->data, GMT->common.o.n_cols, double *);
+				if (!extend) {
+					S->data = gmt_M_memory (GMT, S->data, GMT->common.o.n_cols, double *);
+					SH->alloc_mode = gmt_M_memory (GMT, SH->alloc_mode, GMT->common.o.n_cols, enum GMT_enum_alloc);
+				}
 			}
 			S->n_columns = GMT->common.o.n_cols;	/* New column count */
 		}
@@ -8490,8 +8523,9 @@ void gmt_free_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT **S) {
 	struct GMT_DATASEGMENT *segment = *S;
 	if (!segment) return;	/* Do not try to free NULL pointer */
 	SH = gmt_get_DS_hidden (segment);
-	if (SH->alloc_mode == GMT_ALLOC_INTERNALLY) {	/* Free data GMT allocated */
-		for (col = 0; col < segment->n_columns; col++) gmt_M_free (GMT, segment->data[col]);
+	for (col = 0; col < segment->n_columns; col++) {
+		if (SH->alloc_mode[col] == GMT_ALLOC_INTERNALLY)	/* Free data GMT allocated */
+			gmt_M_free (GMT, segment->data[col]);
 	}
 	gmt_M_free (GMT, segment->data);
 	gmt_M_free (GMT, segment->min);
@@ -8501,7 +8535,8 @@ void gmt_free_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT **S) {
 	for (k = 0; k < 2; k++) gmt_M_str_free (SH->file[k]);
 	if (SH->ogr) gmtio_free_ogr_seg (GMT, segment);	/* OGR metadata */
 	gmtio_free_segment_text (GMT, segment);
-	gmt_M_free (GMT, SH);
+	gmt_M_free (GMT, SH->alloc_mode);
+	gmt_M_free (GMT, segment->hidden);
 	gmt_M_free (GMT, segment);
 	*S = NULL;
 }
@@ -8524,7 +8559,7 @@ void gmt_free_table (struct GMT_CTRL *GMT, struct GMT_DATATABLE *table) {
 		for (seg = 0; seg < table->n_segments; seg++) gmt_free_segment (GMT, &(table->segment[seg]));
 		gmt_M_free (GMT, table->segment);
 	}
-	gmt_M_free (GMT, TH);
+	gmt_M_free (GMT, table->hidden);
 	gmt_M_free (GMT, table);
 }
 
diff --git a/src/gmt_map.c b/src/gmt_map.c
index b25f1b1475e..2b141c9ee31 100644
--- a/src/gmt_map.c
+++ b/src/gmt_map.c
@@ -5183,7 +5183,7 @@ void gmt_wesn_search (struct GMT_CTRL *GMT, double xmin, double xmax, double ymi
 GMT_LOCAL void gmtmap_genper_search (struct GMT_CTRL *GMT, double *west, double *east, double *south, double *north, bool add_pad) {
 	double w, e, s = 90.0, n = -90.0, *work_x = NULL, *work_y = NULL;
 	uint64_t np, k;
-	struct GMT_DATASEGMENT *S = gmt_get_segment (GMT);
+	struct GMT_DATASEGMENT *S = gmt_get_segment (GMT, 2);
 	/* Because the genper clip path may be a mix of straight borders and a curved horizon, we must determine this
 	 * clip path and search along it, getting lon,lat along the way, and find the extreme values to use. Before this
 	 * function was added, we ended up in the gmt_wesn_search function which would fail along the horizon in many
diff --git a/src/gmt_prototypes.h b/src/gmt_prototypes.h
index 6a7e9bea523..a6788b14602 100644
--- a/src/gmt_prototypes.h
+++ b/src/gmt_prototypes.h
@@ -322,7 +322,7 @@ EXTERN_MSC struct GMT_GRID_HEADER * gmt_get_header (struct GMT_CTRL *GMT);
 EXTERN_MSC struct GMT_POSTSCRIPT * gmt_get_postscript (struct GMT_CTRL *GMT);
 EXTERN_MSC struct GMT_DATASET * gmt_get_dataset (struct GMT_CTRL *GMT);
 EXTERN_MSC struct GMT_DATATABLE * gmt_get_table (struct GMT_CTRL *GMT);
-EXTERN_MSC struct GMT_DATASEGMENT * gmt_get_segment (struct GMT_CTRL *GMT);
+EXTERN_MSC struct GMT_DATASEGMENT * gmt_get_segment (struct GMT_CTRL *GMT, uint64_t n_columns);
 EXTERN_MSC int gmt_ascii_output_no_text (struct GMT_CTRL *GMT, FILE *fp, uint64_t n, double *ptr, char *txt);
 EXTERN_MSC void gmt_set_column_type (struct GMT_CTRL *GMT, unsigned int direction, unsigned int col, enum gmt_col_enum type);
 EXTERN_MSC enum gmt_col_enum gmt_get_column_type (struct GMT_CTRL *GMT, unsigned int direction, unsigned int col);
diff --git a/src/gmt_support.c b/src/gmt_support.c
index bf2993eedfe..ac6e4d270c4 100644
--- a/src/gmt_support.c
+++ b/src/gmt_support.c
@@ -3001,10 +3001,11 @@ GMT_LOCAL void gmtsupport_add_decoration (struct GMT_CTRL *GMT, struct GMT_DATAS
 	if (S->n_rows == SH->n_alloc) {	/* Need more memory for the segment */
 		uint64_t col;
 		SH->n_alloc += GMT_SMALL_CHUNK;
-		for (col = 0; col < S->n_columns; col++)
+		for (col = 0; col < S->n_columns; col++) {
 			S->data[col] = gmt_M_memory (GMT, S->data[col], SH->n_alloc, double);
+			SH->alloc_mode[col] = GMT_ALLOC_INTERNALLY;
+		}
 		S->text = gmt_M_memory (GMT, S->text, SH->n_alloc, char *);
-		SH->alloc_mode = GMT_ALLOC_INTERNALLY;
 	}
 	/* Deal with any justifications or nudging */
 	if (G->nudge_flag) {	/* Must adjust point a bit */
@@ -16098,7 +16099,7 @@ unsigned int gmtlib_split_line_at_dateline (struct GMT_CTRL *GMT, struct GMT_DAT
 	uint64_t k, col, seg, row, start, length, *pos = gmt_M_memory (GMT, NULL, S->n_rows, uint64_t);
 	char label[GMT_BUFSIZ] = {""}, *txt = NULL, *feature = "Line";
 	double r;
-	struct GMT_DATASEGMENT **L = NULL, *Sx = gmt_get_segment (GMT);
+	struct GMT_DATASEGMENT **L = NULL, *Sx = gmt_get_segment (GMT, S->n_columns);
 	struct GMT_DATASEGMENT_HIDDEN *LH = NULL, *SH = gmt_get_DS_hidden (S);
 
 	for (k = 0; k < S->n_rows; k++) gmt_lon_range_adjust (GMT_IS_0_TO_P360_RANGE, &S->data[GMT_X][k]);	/* First enforce 0 <= lon < 360 so we don't have to check again */
diff --git a/src/gmtsplit.c b/src/gmtsplit.c
index 4e82e6f8dd5..17c231e4137 100644
--- a/src/gmtsplit.c
+++ b/src/gmtsplit.c
@@ -452,7 +452,7 @@ EXTERN_MSC int GMT_gmtsplit (void *V_API, int mode, void *args) {
 		h_col = no_z_column + 3;
 	}
 	z_cols = 2;
-	S_out = gmt_get_segment (GMT);	/* We will use a single segment to hold all output records and keep track of start/stop via rec[] array */
+	S_out = gmt_get_segment (GMT, n_outputs);	/* We will use a single segment to hold all output records and keep track of start/stop via rec[] array */
 
 	nprofiles = 0;
 	for (tbl = 0; tbl < D[GMT_IN]->n_tables; tbl++) {
diff --git a/src/grdview.c b/src/grdview.c
index fb418fbab00..35955b55216 100644
--- a/src/grdview.c
+++ b/src/grdview.c
@@ -1300,7 +1300,7 @@ EXTERN_MSC int GMT_grdview (void *V_API, int mode, void *args) {
 	if (Ctrl->T.active) {	/* Plot image as polygonal pieces. Here, -JZ is not set */
 		double *xx = NULL, *yy = NULL;
 		struct GMT_FILL fill;
-		struct GMT_DATASEGMENT *S = gmt_get_segment (GMT);
+		struct GMT_DATASEGMENT *S = gmt_get_segment (GMT, 2);
 		gmt_init_fill (GMT, &fill, -1.0, -1.0, -1.0);	/* Initialize fill structure */
 
 		GMT_Report (API, GMT_MSG_INFORMATION, "Tiling without interpolation\n");
@@ -1323,6 +1323,7 @@ EXTERN_MSC int GMT_grdview (void *V_API, int mode, void *args) {
 			gmt_M_free (GMT, xx);
 			gmt_M_free (GMT, yy);
 		}
+		S->data[GMT_X] = S->data[GMT_Y] = NULL;	/* Since xx and yy was set to NULL but not data... */
 		gmt_free_segment (GMT, &S);
 	}
 
diff --git a/src/pslegend.c b/src/pslegend.c
index 799269d4c42..34b71304fa5 100644
--- a/src/pslegend.c
+++ b/src/pslegend.c
@@ -386,12 +386,37 @@ GMT_LOCAL void pslegend_maybe_realloc_table (struct GMT_CTRL *GMT, struct GMT_DA
 	if (k < TH->n_alloc) return;	/* Not yet */
 	T->segment = gmt_M_memory (GMT, T->segment, TH->n_alloc + GMT_SMALL_CHUNK, struct GMT_DATASEGMENT *);
 	for (unsigned int seg = TH->n_alloc; seg < TH->n_alloc + GMT_SMALL_CHUNK; seg++) {
-		T->segment[seg] = gmt_get_segment (GMT);
+		T->segment[seg] = gmt_get_segment (GMT, T->n_columns);
 		gmt_alloc_segment (GMT, T->segment[seg], n_rows, T->n_columns, mode, true);
 	}
 	TH->n_alloc += GMT_SMALL_CHUNK;
 }
 
+GMT_LOCAL void pslegend_free_unused_segments (struct GMT_CTRL *GMT, struct GMT_DATATABLE *T, uint64_t k) {
+	/* Must finalize the number of allocated segments */
+	struct GMT_DATATABLE_HIDDEN *TH = gmt_get_DT_hidden (T);
+	if (k == TH->n_alloc) return;	/* Exact match */
+	T->n_segments = k;
+	for (unsigned int seg = T->n_segments; seg < TH->n_alloc; seg++) {
+		gmt_free_segment (GMT, &(T->segment[seg]));
+	}
+	T->segment = gmt_M_memory (GMT, T->segment, T->n_segments, struct GMT_DATASEGMENT *);
+	TH->n_alloc = T->n_segments;
+}
+
+GMT_LOCAL void pslegend_free_unused_rows (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT *S, uint64_t k) {
+	/* Must finalize the number of allocated rows */
+	struct GMT_DATASEGMENT_HIDDEN *SH = gmt_get_DS_hidden (S);
+	if (k == SH->n_alloc) return;	/* Not yet */
+	S->n_rows = k;
+	if (S->n_columns) {	/* Numerical data */
+		uint64_t col;
+		for (col = 0; col < S->n_columns; col++)
+			S->data[col] = gmt_M_memory (GMT, S->data[col], S->n_rows, double);
+	}
+	if (S->text) S->text = gmt_M_memory (GMT, S->text, S->n_rows, char *);
+}
+
 GMT_LOCAL void pslegend_maybe_realloc_segment (struct GMT_CTRL *GMT, struct GMT_DATASEGMENT *S) {
 	struct GMT_DATASEGMENT_HIDDEN *SH = gmt_get_DS_hidden (S);
 	if (S->n_rows < SH->n_alloc) return;	/* Not yet */
@@ -1824,9 +1849,14 @@ EXTERN_MSC int GMT_pslegend (void *V_API, int mode, void *args) {
 	/* Time to plot any symbols, text, fronts, quoted lines, and paragraphs we collected in the loop */
 
 	if (D[FRONT]) {
-		TH = gmt_get_DT_hidden (D[FRONT]->table[0]);
-		/* Create option list, register D[FRONT] as input source */
-		D[FRONT]->table[0]->n_segments = n_fronts;	/* Set correct number of fronts */
+		pslegend_free_unused_segments (GMT, D[FRONT]->table[0], n_fronts);
+#ifdef DEBUG
+		if (Ctrl->DBG.active) {
+			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_LINE, GMT_IO_RESET, NULL, "dump_front.txt", D[FRONT]) != GMT_NOERROR) {
+				Return (API->error);
+			}
+		}
+#endif
 		if (GMT_Open_VirtualFile (API, GMT_IS_DATASET, GMT_IS_LINE, GMT_IN|GMT_IS_REFERENCE, D[FRONT], string) != GMT_NOERROR) {
 			Return (API->error);
 		}
@@ -1838,19 +1868,17 @@ EXTERN_MSC int GMT_pslegend (void *V_API, int mode, void *args) {
 		if (GMT_Close_VirtualFile (API, string) != GMT_NOERROR) {
 			Return (API->error);
 		}
+	}
+	if (D[QLINE]) {
+		pslegend_free_unused_segments (GMT, D[QLINE]->table[0], n_quoted_lines);
 #ifdef DEBUG
 		if (Ctrl->DBG.active) {
-			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_LINE, GMT_IO_RESET, NULL, "dump_front.txt", D[FRONT]) != GMT_NOERROR) {
+			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_LINE, GMT_IO_RESET, NULL, "dump_qline.txt", D[QLINE]) != GMT_NOERROR) {
 				Return (API->error);
 			}
 		}
 #endif
-		D[FRONT]->table[0]->n_segments = TH->n_alloc;	/* Reset to allocation limit */
-	}
-	if (D[QLINE]) {
-		TH = gmt_get_DT_hidden (D[QLINE]->table[0]);
 		/* Create option list, register D[QLINE] as input source */
-		D[QLINE]->table[0]->n_segments = n_quoted_lines;	/* Set correct number of lines */
 		if (GMT_Open_VirtualFile (API, GMT_IS_DATASET, GMT_IS_LINE, GMT_IN|GMT_IS_REFERENCE, D[QLINE], string) != GMT_NOERROR) {
 			Return (API->error);
 		}
@@ -1862,19 +1890,17 @@ EXTERN_MSC int GMT_pslegend (void *V_API, int mode, void *args) {
 		if (GMT_Close_VirtualFile (API, string) != GMT_NOERROR) {
 			Return (API->error);
 		}
+	}
+	if (D[DLINE]) {
+		pslegend_free_unused_segments (GMT, D[DLINE]->table[0], n_decorated_lines);
 #ifdef DEBUG
 		if (Ctrl->DBG.active) {
-			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_LINE, GMT_IO_RESET, NULL, "dump_qline.txt", D[QLINE]) != GMT_NOERROR) {
+			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_LINE, GMT_IO_RESET, NULL, "dump_dline.txt", D[DLINE]) != GMT_NOERROR) {
 				Return (API->error);
 			}
 		}
 #endif
-		D[QLINE]->table[0]->n_segments = TH->n_alloc;	/* Reset to allocation limit */
-	}
-	if (D[DLINE]) {
-		TH = gmt_get_DT_hidden (D[DLINE]->table[0]);
 		/* Create option list, register D[DLINE] as input source */
-		D[DLINE]->table[0]->n_segments = n_decorated_lines;	/* Set correct number of lines */
 		if (GMT_Open_VirtualFile (API, GMT_IS_DATASET, GMT_IS_LINE, GMT_IN|GMT_IS_REFERENCE, D[DLINE], string) != GMT_NOERROR) {
 			Return (API->error);
 		}
@@ -1886,18 +1912,16 @@ EXTERN_MSC int GMT_pslegend (void *V_API, int mode, void *args) {
 		if (GMT_Close_VirtualFile (API, string) != GMT_NOERROR) {
 			Return (API->error);
 		}
+	}
+	if (D[SYM]) {
+		pslegend_free_unused_segments (GMT, D[SYM]->table[0], n_symbols);
 #ifdef DEBUG
 		if (Ctrl->DBG.active) {
-			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_LINE, GMT_IO_RESET, NULL, "dump_dline.txt", D[DLINE]) != GMT_NOERROR) {
+			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_POINT, GMT_IO_RESET, NULL, "dump_sym.txt", D[SYM]) != GMT_NOERROR) {
 				Return (API->error);
 			}
 		}
 #endif
-		D[DLINE]->table[0]->n_segments = TH->n_alloc;	/* Reset to allocation limit */
-	}
-	if (D[SYM]) {
-		TH = gmt_get_DT_hidden (D[SYM]->table[0]);
-		D[SYM]->table[0]->n_segments = n_symbols;	/* Set correct number of segments */
 		/* Create option list, register D[SYM] as input source */
 		if (GMT_Open_VirtualFile (API, GMT_IS_DATASET, GMT_IS_POINT, GMT_IN|GMT_IS_REFERENCE, D[SYM], string) != GMT_NOERROR) {
 			Return (API->error);
@@ -1911,18 +1935,17 @@ EXTERN_MSC int GMT_pslegend (void *V_API, int mode, void *args) {
 		if (GMT_Close_VirtualFile (API, string) != GMT_NOERROR) {
 			Return (API->error);
 		}
+	}
+	if (D[TXT]) {
+		pslegend_free_unused_rows (GMT, D[TXT]->table[0]->segment[0], krow[TXT]);
+		D[TXT]->n_records = krow[TXT];
 #ifdef DEBUG
 		if (Ctrl->DBG.active) {
-			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_POINT, GMT_IO_RESET, NULL, "dump_sym.txt", D[SYM]) != GMT_NOERROR) {
+			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_NONE, GMT_IO_RESET, NULL, "dump_txt.txt", D[TXT]) != GMT_NOERROR) {
 				Return (API->error);
 			}
 		}
 #endif
-		D[SYM]->table[0]->n_segments = TH->n_alloc;	/* Reset to allocation limit */
-	}
-	if (D[TXT]) {
-		TH = gmt_get_DT_hidden (D[TXT]->table[0]);
-		D[TXT]->table[0]->segment[0]->n_rows = D[TXT]->n_records = krow[TXT];
 		/* Create option list, register D[TXT] as input source */
 		if (GMT_Open_VirtualFile (API, GMT_IS_DATASET, GMT_IS_NONE, GMT_IN|GMT_IS_REFERENCE, D[TXT], string) != GMT_NOERROR) {
 			Return (API->error);
@@ -1935,14 +1958,6 @@ EXTERN_MSC int GMT_pslegend (void *V_API, int mode, void *args) {
 		if (GMT_Close_VirtualFile (API, string) != GMT_NOERROR) {
 			Return (API->error);
 		}
-#ifdef DEBUG
-		if (Ctrl->DBG.active) {
-			if (GMT_Write_Data (API, GMT_IS_DATASET, GMT_IS_FILE, GMT_IS_NONE, GMT_IO_RESET, NULL, "dump_txt.txt", D[TXT]) != GMT_NOERROR) {
-				Return (API->error);
-			}
-		}
-#endif
-		D[TXT]->table[0]->segment[0]->n_rows = D[TXT]->n_records = TH->n_alloc;	/* To free what we allocated */
 	}
 	if (D[PAR]) {
 		if (n_para >= 0) {	/* End of last paragraph for sure */
diff --git a/src/sphdistance.c b/src/sphdistance.c
index 21e18d14c71..404f17a44bd 100644
--- a/src/sphdistance.c
+++ b/src/sphdistance.c
@@ -523,7 +523,7 @@ EXTERN_MSC int GMT_sphdistance (void *V_API, int mode, void *args) {
 		}
 		else {	/* Obtain current polygon from Voronoi listings */
 			if (P == NULL) {	/* Need a single polygon structure that we reuse for each polygon */
-				P = gmt_get_segment (GMT);	/* Needed as pointer below */
+				P = gmt_get_segment (GMT, 2);	/* Needed as pointer below */
 				P->data = gmt_M_memory (GMT, NULL, 2, double *);	/* Needed as pointers below */
 				P->min = gmt_M_memory (GMT, NULL, 2, double);	/* Needed to hold min lon/lat */
 				P->max = gmt_M_memory (GMT, NULL, 2, double);	/* Needed to hold max lon/lat */
diff --git a/src/spotter/grdrotater.c b/src/spotter/grdrotater.c
index c12c822a609..51105ca0aef 100644
--- a/src/spotter/grdrotater.c
+++ b/src/spotter/grdrotater.c
@@ -397,7 +397,6 @@ GMT_LOCAL struct GMT_DATASET * grdrotater_get_grid_path (struct GMT_CTRL *GMT, s
 	S->min[GMT_Y] = h->wesn[YLO];	S->max[GMT_Y] = h->wesn[YHI];
 	SH = gmt_get_DS_hidden (S);
 	SH->pole = 0;
-	SH->alloc_mode = GMT_ALLOC_INTERNALLY;
 
 	return (D);
 }
diff --git a/src/testapi_uservectors.c b/src/testapi_uservectors.c
index 2716ed9e48a..4d364cb1906 100644
--- a/src/testapi_uservectors.c
+++ b/src/testapi_uservectors.c
@@ -220,7 +220,7 @@ int deploy_test (unsigned int intype, unsigned int outtype, int alloc_in_GMT, in
 	/* Hook the user input arrays up to this container */
 	GMT_Put_Vector (API, V[GMT_IN], GMT_X, intype, in_data[GMT_X]);
 	GMT_Put_Vector (API, V[GMT_IN], GMT_Y, intype, in_data[GMT_Y]);
-	/* Associate our vectors container with a virtual dataset file to "read" from */
+	/* Associate our vectors container with a virtual dataset file to "read" from. REFERENCE will be switched to DUPLICATE if input is not double  */
 	GMT_Open_VirtualFile (API, GMT_IS_DATASET|GMT_VIA_VECTOR, GMT_IS_POINT, GMT_IN|GMT_IS_REFERENCE, V[GMT_IN], input);
 	if (alloc_in_GMT)	/* Request vectors container for output data to be allocated by GMT */
 		GMT_Open_VirtualFile (API, GMT_IS_DATASET|GMT_VIA_VECTOR, out_via, GMT_OUT|GMT_IS_REFERENCE, NULL, output);
@@ -237,8 +237,8 @@ int deploy_test (unsigned int intype, unsigned int outtype, int alloc_in_GMT, in
 		/* Hook the user output array up to this containers */
 		GMT_Put_Vector (API, V[GMT_OUT], GMT_X, outtype, out_data[GMT_X]);
 		GMT_Put_Vector (API, V[GMT_OUT], GMT_Y, outtype, out_data[GMT_Y]);
-   		/* Associate our data vectors with a virtual dataset file to "write" to */
-    		GMT_Open_VirtualFile (API, GMT_IS_DATASET|GMT_VIA_VECTOR, GMT_IS_POINT, GMT_OUT, V[GMT_OUT], output);
+   		/* Associate our data vectors with a virtual dataset file to "write" to. REFERENCE will be switched to DUPLICATE if output is not double */
+    	GMT_Open_VirtualFile (API, GMT_IS_DATASET|GMT_VIA_VECTOR, GMT_IS_POINT, GMT_OUT|GMT_IS_REFERENCE, V[GMT_OUT], output);
 	}
 	/* Prepare the module arguments to multiply the input dataset by 10 then add 1 */
 	sprintf (args, "%s 10 MUL 1 ADD = %s", input, output);