From 42476b370e1cb30786ea02155cbc687c06a124bf Mon Sep 17 00:00:00 2001
From: Fred <98240+farnoux@users.noreply.github.com>
Date: Wed, 8 Jan 2025 10:39:25 +0100
Subject: [PATCH] Handle data when > 6000 rows

At the time of the commit PostgREST limit is 6000.

But I choose to cut pagination by pages of 5000, allowing for maximum 10000 rows in total. Because 10000 rows is the maximum limit on Airtable sync API.

When we'll have 10000+ users, I hope this edge function will have been replaced by another reliable method.
---
 supabase/functions/crm_sync/index.ts | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/supabase/functions/crm_sync/index.ts b/supabase/functions/crm_sync/index.ts
index 4f1bff2cab..481233fc69 100644
--- a/supabase/functions/crm_sync/index.ts
+++ b/supabase/functions/crm_sync/index.ts
@@ -21,7 +21,23 @@ serve(async (req) => {
   const { token, table, sync } = await req.json();
 
   // Récupère les données de la table au format csv.
-  const csvResponse = await supabase.from(table).select().csv();
+  const { error, data, count } = await supabase
+    .from(table)
+    .select('*', { count: 'exact' })
+    .range(0, 4999)
+    .csv();
+
+  let allData = data;
+
+  if (count > 5000) {
+    const { data: dataOver5000 } = await supabase
+      .from(table)
+      .select('*')
+      .range(5000, 9999)
+      .csv();
+
+    allData += '\n' + dataOver5000.replace(/.*\n/, '') /* remove header */;
+  }
 
   // Envoie les données à Airtable vers l'endpoint sync.
   const syncResponse = await fetch(sync, {
@@ -30,10 +46,17 @@ serve(async (req) => {
       Authorization: `Bearer ${token}`,
       'Content-Type': 'text/csv',
     },
-    body: csvResponse.data,
+    body: allData,
   });
 
   const { success } = await syncResponse.json();
+  console.log(
+    JSON.stringify({
+      success,
+      table,
+      nbOfRows: allData.split('\n').length - 1 /* without header */,
+    })
+  );
 
   // Renvoie le statut pour persister dans les logs Supabase.
   return new Response(