From f43aa609e018007950383dd71d3d846cf79f8abf Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 18 Aug 2023 16:25:16 -0500 Subject: [PATCH 1/4] Add java API to get size of host memory needed to copy column view Co-authored-by: Jim Brennan --- .../main/java/ai/rapids/cudf/ColumnView.java | 52 +++++++++++++++++++ .../java/ai/rapids/cudf/ColumnVectorTest.java | 8 +++ 2 files changed, 60 insertions(+) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 0a7346d1cbc..302ce8d6896 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -5160,6 +5160,58 @@ public HostColumnVector copyToHost() { } } + /** + * Calculate the total space required to copy the data to the host. + */ + public long getHostBytesRequired() { + return getHostBytesRequiredHelper(this); + } + + /* + * Align given size to account for host allocation alignment of 64 + */ + private static long alignAllocSize(long size) { + final long align = 64; // must be a power of two + return (size + (align - 1)) & ~(align - 1); + } + + private static long getHostBytesRequiredHelper( + ColumnView deviceCvPointer) { + if (deviceCvPointer == null) { + return 0; + } + BaseDeviceMemoryBuffer valid = deviceCvPointer.getValid(); + BaseDeviceMemoryBuffer offsets = deviceCvPointer.getOffsets(); + BaseDeviceMemoryBuffer data = null; + DType type = deviceCvPointer.getType(); + if (!type.isNestedType()) { + data = deviceCvPointer.getData(); + } + long validityLength = 0; + long offsetsLength = 0; + long dataLength = 0; + long childrenLength = 0; + if (valid != null) { + validityLength = alignAllocSize(valid.getLength()); + } + if (offsets != null) { + offsetsLength = alignAllocSize(offsets.getLength()); + } + // If a strings column is all null values there is no data buffer allocated + if (data != null) { + dataLength = alignAllocSize(data.length); + data.close(); + } + if (type.isNestedType()) { + for (int i = 0; i < deviceCvPointer.getNumChildren(); i++) { + try (ColumnView childDevPtr = deviceCvPointer.getChildColumnView(i)) { + childrenLength += getHostBytesRequiredHelper(childDevPtr); + } + } + } + return validityLength + offsetsLength + dataLength + childrenLength; + } + /** * Exact check if a column or its descendants have non-empty null rows * diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 0e1fbad6129..b6f0d3708d0 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -1031,7 +1031,9 @@ void testGetDeviceMemorySizeNonStrings() { try (ColumnVector v0 = ColumnVector.fromBoxedInts(1, 2, 3, 4, 5, 6); ColumnVector v1 = ColumnVector.fromBoxedInts(1, 2, 3, null, null, 4, 5, 6)) { assertEquals(24, v0.getDeviceMemorySize()); // (6*4B) + assertEquals(64, v0.getHostBytesRequired()); // account for alignment padding assertEquals(96, v1.getDeviceMemorySize()); // (8*4B) + 64B(for validity vector) + assertEquals(64 + 64, v1.getHostBytesRequired()); } } @@ -1040,7 +1042,9 @@ void testGetDeviceMemorySizeStrings() { try (ColumnVector v0 = ColumnVector.fromStrings("onetwothree", "four", "five"); ColumnVector v1 = ColumnVector.fromStrings("onetwothree", "four", null, "five")) { assertEquals(35, v0.getDeviceMemorySize()); //19B data + 4*4B offsets = 35 + assertEquals(64 + 64, v0.getHostBytesRequired()); // account for alignment padding assertEquals(103, v1.getDeviceMemorySize()); //19B data + 5*4B + 64B validity vector = 103B + assertEquals(64+64+64, v1.getHostBytesRequired()); // account for alignment padding } } @@ -1064,10 +1068,12 @@ void testGetDeviceMemorySizeLists() { // 24 bytes for offsets of of string column // 22 bytes of string character size assertEquals(64+16+64+24+22, sv.getDeviceMemorySize()); + assertEquals(64+64+64+64+64, sv.getHostBytesRequired()); // account for alignment padding // 20 bytes for offsets of list column // 28 bytes for data of INT32 column assertEquals(20+28, iv.getDeviceMemorySize()); + assertEquals(64+64, iv.getHostBytesRequired()); // account for alignment padding } } @@ -1096,6 +1102,8 @@ void testGetDeviceMemorySizeStructs() { // 64 bytes for validity of int64 column // 28 bytes for data of the int64 column assertEquals(64+64+20+64+28+22+64+28, v.getDeviceMemorySize()); + // account for alignment padding + assertEquals(64+64+64+64+64+64+64+64, v.getHostBytesRequired()); } } From 6886a551182eb679072313d2ea7664b0727252eb Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Mon, 21 Aug 2023 10:50:41 -0500 Subject: [PATCH 2/4] Moved size calculation to C++ --- .../main/java/ai/rapids/cudf/ColumnView.java | 59 ++++--------------- java/src/main/native/src/ColumnViewJni.cpp | 22 +++++-- 2 files changed, 27 insertions(+), 54 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 302ce8d6896..8913a444e0e 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -307,7 +307,15 @@ public final int getNumChildren() { * Returns the amount of device memory used. */ public long getDeviceMemorySize() { - return getDeviceMemorySize(getNativeView()); + return getDeviceMemorySize(getNativeView(), false); + } + + /** + * Returns the amount of memory used by this, but padded for 64-bit alignment. This makes it + * so it could be used as the amount of memory needed to copy the data to the host. + */ + public long getDeviceMemorySizeAligned() { + return getDeviceMemorySize(getNativeView(), true); } @Override @@ -4789,7 +4797,7 @@ static native long makeCudfColumnView(int type, int scale, long data, long dataS static native int getNativeNumChildren(long viewHandle) throws CudfException; // calculate the amount of device memory used by this column including any child columns - static native long getDeviceMemorySize(long viewHandle) throws CudfException; + static native long getDeviceMemorySize(long viewHandle, boolean aligned) throws CudfException; static native long copyColumnViewToCV(long viewHandle) throws CudfException; @@ -5164,52 +5172,7 @@ public HostColumnVector copyToHost() { * Calculate the total space required to copy the data to the host. */ public long getHostBytesRequired() { - return getHostBytesRequiredHelper(this); - } - - /* - * Align given size to account for host allocation alignment of 64 - */ - private static long alignAllocSize(long size) { - final long align = 64; // must be a power of two - return (size + (align - 1)) & ~(align - 1); - } - - private static long getHostBytesRequiredHelper( - ColumnView deviceCvPointer) { - if (deviceCvPointer == null) { - return 0; - } - BaseDeviceMemoryBuffer valid = deviceCvPointer.getValid(); - BaseDeviceMemoryBuffer offsets = deviceCvPointer.getOffsets(); - BaseDeviceMemoryBuffer data = null; - DType type = deviceCvPointer.getType(); - if (!type.isNestedType()) { - data = deviceCvPointer.getData(); - } - long validityLength = 0; - long offsetsLength = 0; - long dataLength = 0; - long childrenLength = 0; - if (valid != null) { - validityLength = alignAllocSize(valid.getLength()); - } - if (offsets != null) { - offsetsLength = alignAllocSize(offsets.getLength()); - } - // If a strings column is all null values there is no data buffer allocated - if (data != null) { - dataLength = alignAllocSize(data.length); - data.close(); - } - if (type.isNestedType()) { - for (int i = 0; i < deviceCvPointer.getNumChildren(); i++) { - try (ColumnView childDevPtr = deviceCvPointer.getChildColumnView(i)) { - childrenLength += getHostBytesRequiredHelper(childDevPtr); - } - } - } - return validityLength + offsetsLength + dataLength + childrenLength; + return getDeviceMemorySizeAligned(); } /** diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index 1cb51a22bf3..dc3d65756b4 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -91,22 +91,31 @@ using cudf::jni::release_as_jlong; namespace { -std::size_t calc_device_memory_size(cudf::column_view const &view) { +std::size_t align_size(std::size_t size, bool const do_it) { + if (do_it) { + constexpr std::size_t ALIGN = 1 << 6; // 64-bit alignment + return (size + (ALIGN - 1)) & ~(ALIGN - 1); + } else { + return size; + } +} + +std::size_t calc_device_memory_size(cudf::column_view const &view, bool const aligned) { std::size_t total = 0; auto row_count = view.size(); if (view.nullable()) { - total += cudf::bitmask_allocation_size_bytes(row_count); + total += align_size(cudf::bitmask_allocation_size_bytes(row_count), aligned); } auto dtype = view.type(); if (cudf::is_fixed_width(dtype)) { - total += cudf::size_of(dtype) * view.size(); + total += align_size(cudf::size_of(dtype) * view.size(), aligned); } return std::accumulate( view.child_begin(), view.child_end(), total, - [](std::size_t t, cudf::column_view const &v) { return t + calc_device_memory_size(v); }); + [aligned](std::size_t t, cudf::column_view const &v) { return t + calc_device_memory_size(v, aligned); }); } } // anonymous namespace @@ -2217,12 +2226,13 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getNativeValidityLength(J } JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getDeviceMemorySize(JNIEnv *env, jclass, - jlong handle) { + jlong handle, + jboolean aligned) { JNI_NULL_CHECK(env, handle, "native handle is null", 0); try { cudf::jni::auto_set_device(env); auto view = reinterpret_cast(handle); - return calc_device_memory_size(*view); + return calc_device_memory_size(*view, aligned); } CATCH_STD(env, 0); } From b685df47587d412b80eecfb18204ecf0b6270e12 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Mon, 21 Aug 2023 13:08:49 -0500 Subject: [PATCH 3/4] Fixed nits --- .../main/java/ai/rapids/cudf/ColumnView.java | 10 ++++----- java/src/main/native/src/ColumnViewJni.cpp | 21 ++++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 8913a444e0e..16106def431 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -311,10 +311,10 @@ public long getDeviceMemorySize() { } /** - * Returns the amount of memory used by this, but padded for 64-bit alignment. This makes it - * so it could be used as the amount of memory needed to copy the data to the host. + * Returns the amount of memory used by this, but padded to the next 64-bit boundary. This + * allows the returned value to also indicate the size needed to copy the data to host memory. */ - public long getDeviceMemorySizeAligned() { + public long getDeviceMemorySizePadded64() { return getDeviceMemorySize(getNativeView(), true); } @@ -4797,7 +4797,7 @@ static native long makeCudfColumnView(int type, int scale, long data, long dataS static native int getNativeNumChildren(long viewHandle) throws CudfException; // calculate the amount of device memory used by this column including any child columns - static native long getDeviceMemorySize(long viewHandle, boolean aligned) throws CudfException; + static native long getDeviceMemorySize(long viewHandle, boolean shouldPad) throws CudfException; static native long copyColumnViewToCV(long viewHandle) throws CudfException; @@ -5172,7 +5172,7 @@ public HostColumnVector copyToHost() { * Calculate the total space required to copy the data to the host. */ public long getHostBytesRequired() { - return getDeviceMemorySizeAligned(); + return getDeviceMemorySizePadded64(); } /** diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index dc3d65756b4..139db278c0e 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -91,8 +91,8 @@ using cudf::jni::release_as_jlong; namespace { -std::size_t align_size(std::size_t size, bool const do_it) { - if (do_it) { +std::size_t pad_size(std::size_t size, bool const should_pad_64) { + if (should_pad_64) { constexpr std::size_t ALIGN = 1 << 6; // 64-bit alignment return (size + (ALIGN - 1)) & ~(ALIGN - 1); } else { @@ -100,22 +100,23 @@ std::size_t align_size(std::size_t size, bool const do_it) { } } -std::size_t calc_device_memory_size(cudf::column_view const &view, bool const aligned) { +std::size_t calc_device_memory_size(cudf::column_view const &view, bool const padded) { std::size_t total = 0; auto row_count = view.size(); if (view.nullable()) { - total += align_size(cudf::bitmask_allocation_size_bytes(row_count), aligned); + total += pad_size(cudf::bitmask_allocation_size_bytes(row_count), padded); } auto dtype = view.type(); if (cudf::is_fixed_width(dtype)) { - total += align_size(cudf::size_of(dtype) * view.size(), aligned); + total += pad_size(cudf::size_of(dtype) * view.size(), padded); } - return std::accumulate( - view.child_begin(), view.child_end(), total, - [aligned](std::size_t t, cudf::column_view const &v) { return t + calc_device_memory_size(v, aligned); }); + return std::accumulate(view.child_begin(), view.child_end(), total, + [padded](std::size_t t, cudf::column_view const &v) { + return t + calc_device_memory_size(v, padded); + }); } } // anonymous namespace @@ -2227,12 +2228,12 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getNativeValidityLength(J JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getDeviceMemorySize(JNIEnv *env, jclass, jlong handle, - jboolean aligned) { + jboolean padded) { JNI_NULL_CHECK(env, handle, "native handle is null", 0); try { cudf::jni::auto_set_device(env); auto view = reinterpret_cast(handle); - return calc_device_memory_size(*view, aligned); + return calc_device_memory_size(*view, padded); } CATCH_STD(env, 0); } From 5ede07214bf851e6bc045dd39cc56d4592de88d9 Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Mon, 21 Aug 2023 15:23:11 -0500 Subject: [PATCH 4/4] Updated tests and alignment to be automatic --- .../main/java/ai/rapids/cudf/ColumnView.java | 20 ++++----- java/src/main/native/src/ColumnViewJni.cpp | 24 ++++++----- .../java/ai/rapids/cudf/ColumnVectorTest.java | 41 +++++++++++-------- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 16106def431..7db40278d4e 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -310,14 +310,6 @@ public long getDeviceMemorySize() { return getDeviceMemorySize(getNativeView(), false); } - /** - * Returns the amount of memory used by this, but padded to the next 64-bit boundary. This - * allows the returned value to also indicate the size needed to copy the data to host memory. - */ - public long getDeviceMemorySizePadded64() { - return getDeviceMemorySize(getNativeView(), true); - } - @Override public void close() { // close the view handle so long as offHeap is not going to do it for us. @@ -4797,7 +4789,7 @@ static native long makeCudfColumnView(int type, int scale, long data, long dataS static native int getNativeNumChildren(long viewHandle) throws CudfException; // calculate the amount of device memory used by this column including any child columns - static native long getDeviceMemorySize(long viewHandle, boolean shouldPad) throws CudfException; + static native long getDeviceMemorySize(long viewHandle, boolean shouldPadForCpu) throws CudfException; static native long copyColumnViewToCV(long viewHandle) throws CudfException; @@ -5169,12 +5161,18 @@ public HostColumnVector copyToHost() { } /** - * Calculate the total space required to copy the data to the host. + * Calculate the total space required to copy the data to the host. This should be padded to + * the alignment that the CPU requires. */ public long getHostBytesRequired() { - return getDeviceMemorySizePadded64(); + return getDeviceMemorySize(getNativeView(), true); } + /** + * Get the size that the host will align memory allocations to in bytes. + */ + public static native long hostPaddingSizeInBytes(); + /** * Exact check if a column or its descendants have non-empty null rows * diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index 139db278c0e..d5aad03645f 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -91,31 +91,31 @@ using cudf::jni::release_as_jlong; namespace { -std::size_t pad_size(std::size_t size, bool const should_pad_64) { - if (should_pad_64) { - constexpr std::size_t ALIGN = 1 << 6; // 64-bit alignment +std::size_t pad_size(std::size_t size, bool const should_pad_for_cpu) { + if (should_pad_for_cpu) { + constexpr std::size_t ALIGN = sizeof(std::max_align_t); return (size + (ALIGN - 1)) & ~(ALIGN - 1); } else { return size; } } -std::size_t calc_device_memory_size(cudf::column_view const &view, bool const padded) { +std::size_t calc_device_memory_size(cudf::column_view const &view, bool const pad_for_cpu) { std::size_t total = 0; auto row_count = view.size(); if (view.nullable()) { - total += pad_size(cudf::bitmask_allocation_size_bytes(row_count), padded); + total += pad_size(cudf::bitmask_allocation_size_bytes(row_count), pad_for_cpu); } auto dtype = view.type(); if (cudf::is_fixed_width(dtype)) { - total += pad_size(cudf::size_of(dtype) * view.size(), padded); + total += pad_size(cudf::size_of(dtype) * view.size(), pad_for_cpu); } return std::accumulate(view.child_begin(), view.child_end(), total, - [padded](std::size_t t, cudf::column_view const &v) { - return t + calc_device_memory_size(v, padded); + [pad_for_cpu](std::size_t t, cudf::column_view const &v) { + return t + calc_device_memory_size(v, pad_for_cpu); }); } @@ -2228,16 +2228,20 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getNativeValidityLength(J JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_getDeviceMemorySize(JNIEnv *env, jclass, jlong handle, - jboolean padded) { + jboolean pad_for_cpu) { JNI_NULL_CHECK(env, handle, "native handle is null", 0); try { cudf::jni::auto_set_device(env); auto view = reinterpret_cast(handle); - return calc_device_memory_size(*view, padded); + return calc_device_memory_size(*view, pad_for_cpu); } CATCH_STD(env, 0); } +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_hostPaddingSizeInBytes(JNIEnv *env, jclass) { + return sizeof(std::max_align_t); +} + JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_clamper(JNIEnv *env, jobject j_object, jlong handle, jlong j_lo_scalar, jlong j_lo_replace_scalar, diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index b6f0d3708d0..1062a765800 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -1026,25 +1026,36 @@ void decimal128Cv() { } } + static final long HOST_ALIGN_BYTES = ColumnView.hostPaddingSizeInBytes(); + + static void assertHostAligned(long expectedDeviceSize, ColumnView cv) { + long deviceSize = cv.getDeviceMemorySize(); + assertEquals(expectedDeviceSize, deviceSize); + long hostSize = cv.getHostBytesRequired(); + assert(hostSize >= deviceSize); + long roundedHostSize = (hostSize / HOST_ALIGN_BYTES) * HOST_ALIGN_BYTES; + assertEquals(hostSize, roundedHostSize, "The host size should be a multiple of " + + HOST_ALIGN_BYTES); + } + @Test void testGetDeviceMemorySizeNonStrings() { try (ColumnVector v0 = ColumnVector.fromBoxedInts(1, 2, 3, 4, 5, 6); ColumnVector v1 = ColumnVector.fromBoxedInts(1, 2, 3, null, null, 4, 5, 6)) { - assertEquals(24, v0.getDeviceMemorySize()); // (6*4B) - assertEquals(64, v0.getHostBytesRequired()); // account for alignment padding - assertEquals(96, v1.getDeviceMemorySize()); // (8*4B) + 64B(for validity vector) - assertEquals(64 + 64, v1.getHostBytesRequired()); + assertHostAligned(24, v0); // (6*4B) + assertHostAligned(96, v1); // (8*4B) + 64B(for validity vector) } } @Test void testGetDeviceMemorySizeStrings() { + if (ColumnView.hostPaddingSizeInBytes() != 8) { + System.err.println("HOST PADDING SIZE: " + ColumnView.hostPaddingSizeInBytes()); + } try (ColumnVector v0 = ColumnVector.fromStrings("onetwothree", "four", "five"); ColumnVector v1 = ColumnVector.fromStrings("onetwothree", "four", null, "five")) { - assertEquals(35, v0.getDeviceMemorySize()); //19B data + 4*4B offsets = 35 - assertEquals(64 + 64, v0.getHostBytesRequired()); // account for alignment padding - assertEquals(103, v1.getDeviceMemorySize()); //19B data + 5*4B + 64B validity vector = 103B - assertEquals(64+64+64, v1.getHostBytesRequired()); // account for alignment padding + assertHostAligned(35, v0); //19B data + 4*4B offsets = 35 + assertHostAligned(103, v1); //19B data + 5*4B + 64B validity vector = 103B } } @@ -1065,15 +1076,13 @@ void testGetDeviceMemorySizeLists() { // 64 bytes for validity of list column // 16 bytes for offsets of list column // 64 bytes for validity of string column - // 24 bytes for offsets of of string column + // 24 bytes for offsets of string column // 22 bytes of string character size - assertEquals(64+16+64+24+22, sv.getDeviceMemorySize()); - assertEquals(64+64+64+64+64, sv.getHostBytesRequired()); // account for alignment padding + assertHostAligned(64+16+64+24+22, sv); // 20 bytes for offsets of list column // 28 bytes for data of INT32 column - assertEquals(20+28, iv.getDeviceMemorySize()); - assertEquals(64+64, iv.getHostBytesRequired()); // account for alignment padding + assertHostAligned(20+28, iv); } } @@ -1097,13 +1106,11 @@ void testGetDeviceMemorySizeStructs() { // 64 bytes for validity of list column // 20 bytes for offsets of list column // 64 bytes for validity of string column - // 28 bytes for offsets of of string column + // 28 bytes for offsets of string column // 22 bytes of string character size // 64 bytes for validity of int64 column // 28 bytes for data of the int64 column - assertEquals(64+64+20+64+28+22+64+28, v.getDeviceMemorySize()); - // account for alignment padding - assertEquals(64+64+64+64+64+64+64+64, v.getHostBytesRequired()); + assertHostAligned(64+64+20+64+28+22+64+28, v); } }