Add newline to csv importer insert sql (#4172)

* add newline to csv importer insert sql * add test * changelog * fix test typo * fix test typo x2
plausible · Jun 3, 2024 · 8588776 · 8588776
1 parent cb074d8
commit 8588776
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,8 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- Fix CSV import by adding a newline to the INSERT statement plausible/analytics#4172
+
 ## v2.1.0 - 2024-05-23
 
 ### Added

diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
@@ -122,7 +122,7 @@ defmodule Plausible.Imported.CSVImporter do
             SELECT {site_id:UInt64}, {import_id:UInt64}, * \
             FROM input({input_structure:String}) \
             WHERE date >= {start_date:Date} AND date <= {end_date:Date} \
-            FORMAT CSVWithNames\
+            FORMAT CSVWithNames
             """
 
           params = %{

diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
@@ -362,6 +362,60 @@ defmodule Plausible.Imported.CSVImporterTest do
       assert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99
     end
 
+    test "accepts cells without quotes", %{site: site, user: user} = ctx do
+      _ = ctx
+
+      csvs = [
+        %{
+          name: "imported_visitors_20111225_20111230.csv",
+          body: """
+          date,visitors,pageviews,bounces,visits,visit_duration
+          2011-12-25,5,50,2,7,8640
+          2011-12-26,3,4,2,3,43
+          2011-12-27,3,6,2,4,2313
+          2011-12-28,6,30,4,8,2264
+          2011-12-29,4,8,5,6,136
+          2011-12-30,1,1,1,1,0
+          """
+        }
+      ]
+
+      uploads =
+        for %{name: name, body: body} <- csvs do
+          on_ee do
+            %{s3_url: s3_url} = Plausible.S3.import_presign_upload(site.id, name)
+            [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)
+            ExAws.request!(ExAws.S3.put_object(bucket, key, body))
+            %{"filename" => name, "s3_url" => s3_url}
+          else
+            local_path = Path.join(ctx.tmp_dir, name)
+            File.write!(local_path, body)
+            %{"filename" => name, "local_path" => local_path}
+          end
+        end
+
+      date_range = CSVImporter.date_range(uploads)
+
+      {:ok, _job} =
+        CSVImporter.new_import(site, user,
+          start_date: date_range.first,
+          end_date: date_range.last,
+          uploads: uploads,
+          storage: on_ee(do: "s3", else: "local")
+        )
+
+      assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety?: false)
+
+      assert %SiteImport{
+               start_date: ~D[2011-12-25],
+               end_date: ~D[2011-12-30],
+               source: :csv,
+               status: :completed
+             } = Repo.get_by!(SiteImport, site_id: site.id)
+
+      assert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99
+    end
+
     test "fails on invalid CSV", %{site: site, user: user} = ctx do
       _ = ctx