From d81b6e6f1fcb3674c190a2e14f01b0cddf8ad69c Mon Sep 17 00:00:00 2001
From: Mathew Cleveland <clevelam@lanl.gov>
Date: Fri, 20 Aug 2021 11:08:58 -0600
Subject: [PATCH] make origin and target write size match in the mpi_put call
 (#1117)

* make origin and target write size match in the mpi_put call

* fix data type

* fix index

Co-authored-by: Cleveland <cleveland@lanl.gov>
---
 src/kde/quick_index.cc | 15 ++++-----------
 src/kde/quick_index.hh |  4 ++--
 2 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/kde/quick_index.cc b/src/kde/quick_index.cc
index 2c627a546..86886cf33 100644
--- a/src/kde/quick_index.cc
+++ b/src/kde/quick_index.cc
@@ -3,7 +3,7 @@
  * \file   kde/quick_index.cc
  * \author Mathew Cleveland
  * \brief  Explicitly defined quick_index functions.
- * \note   Copyright (C) 2021 Triad National Security, LLC., All rights reserved. 
+ * \note   Copyright (C) 2021-2021 Triad National Security, LLC., All rights reserved.
  */
 //------------------------------------------------------------------------------------------------//
 
@@ -141,11 +141,6 @@ quick_index::quick_index(const size_t dim_, const std::vector<std::array<double,
     }
     rtt_c4::global_sum(&global_need_bins_per_proc[0], nbins * nodes);
 
-    // Build a global list of buffer sizes
-    std::vector<int> proc_ghost_buffer_size(nodes, 0);
-    proc_ghost_buffer_size[node] = static_cast<int>(local_ghost_buffer_size);
-    rtt_c4::global_sum(&proc_ghost_buffer_size[0], nodes);
-
     // calculate the put map so each node knows which processor to send data
     // and where to index that data
     // PERFORMANCE NOTE: This would be more efficient to use a MPI_SCAN and
@@ -172,8 +167,7 @@ quick_index::quick_index(const size_t dim_, const std::vector<std::array<double,
               max_put_buffer_size = map.second.size();
 
             // build up map data
-            put_window_map[map.first].push_back(
-                std::array<int, 3>{rec_proc, proc_ghost_buffer_size[rec_proc], offset});
+            put_window_map[map.first].push_back(std::array<int, 2>{rec_proc, offset});
             offset += static_cast<int>(map.second.size());
           }
         }
@@ -197,8 +191,7 @@ auto put_lambda = [](auto &put, auto &put_buffer, auto &put_size, auto &win) {
   // loop over all ranks we need to send this buffer too.
   for (auto &putv : put.second) {
     const int put_rank = putv[0];
-    const int put_rank_buffer_size = putv[1];
-    const int put_offset = putv[2];
+    const int put_offset = putv[1];
     // This is dumb, but we need to write in chunks because MPI_Put writes
     // junk with large (>10,000) buffer sizes.
     int chunk_size = 1000;
@@ -208,7 +201,7 @@ auto put_lambda = [](auto &put, auto &put_buffer, auto &put_size, auto &win) {
     for (int c = 0; c < nchunks; c++) {
       chunk_size = std::min(chunk_size, static_cast<int>(put_size) - nput);
       Check(chunk_size > 0);
-      MPI_Put(&put_buffer[nput], chunk_size, MPI_DOUBLE, put_rank, put_offset, put_rank_buffer_size,
+      MPI_Put(&put_buffer[nput], chunk_size, MPI_DOUBLE, put_rank, put_offset, chunk_size,
               MPI_DOUBLE, win);
       nput += chunk_size;
     }
diff --git a/src/kde/quick_index.hh b/src/kde/quick_index.hh
index dc87e281f..b9d8815f0 100644
--- a/src/kde/quick_index.hh
+++ b/src/kde/quick_index.hh
@@ -5,7 +5,7 @@
  * \brief  This class generates coarse spatial indexing to quickly access near-neighbor data. This
  *         additionally provides simple interpolation schemes to map data to simple structured
  *         meshes. 
- * \note   Copyright (C) 2021 Triad National Security, LLC., All rights reserved. 
+ * \note   Copyright (C) 2021-2021 Triad National Security, LLC., All rights reserved.
  */
 //------------------------------------------------------------------------------------------------//
 
@@ -130,7 +130,7 @@ private:
   // Map used to write local data to other processor ghost cells
   // put_window_map[global_id] = [put_rank, ghost_proc_buffer_size, ghost_proc_put_offset]
   // array is integers to accommodate mpi data types
-  std::map<size_t, std::vector<std::array<int, 3>>> put_window_map;
+  std::map<size_t, std::vector<std::array<int, 2>>> put_window_map;
   // max put buffer size;
   size_t max_put_buffer_size;
   const double pi = rtt_units::PI;