Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Go] Fatal Error in pqarrow.writeDenseArrow : invalid pointer #28

Open
Niivii opened this issue Aug 7, 2024 · 17 comments
Open

[Go] Fatal Error in pqarrow.writeDenseArrow : invalid pointer #28

Niivii opened this issue Aug 7, 2024 · 17 comments
Labels
Type: bug Something isn't working

Comments

@Niivii
Copy link

Niivii commented Aug 7, 2024 •
edited
Loading

Got a runtine error for this code in Python with polars and adbc engine to Snowflake:

with adbc_driver_snowflake.dbapi.connect(db_kwargs={
    "username": user,
    "adbc.snowflake.sql.account": host,
    "adbc.snowflake.sql.auth_type": "auth_ext_browser",
    "adbc.snowflake.sql.warehouse": warehouse,
    "adbc.snowflake.sql.role": role,
    "adbc.snowflake.sql.db": database,
    "adbc.snowflake.sql.schema": db_schema,
}) as connection:
    self.write_database(
        table_name=table,
        connection=connection,
        if_table_exists="replace",
        engine="adbc",
    )

Stacktrace:

runtime: bad pointer in frame github.com/apache/arrow/go/v17/parquet/pqarrow.writeDenseArrow at 0xc00100d5e0: 0x1
fatal error: invalid pointer found on stack

runtime stack:
runtime.throw({0x7fa6b183582f?, 0x7fa6b2daf908?})
	/usr/local/go/src/runtime/panic.go:1077 +0x5e fp=0x7fa6af8ea590 sp=0x7fa6af8ea560 pc=0x7fa6afdba69e
runtime.adjustpointers(0x7fa6af8ea808?, 0x7fa6af8ea650, 0x7fa6afdddd65?, {0x7fa6af8ea808?, 0x0?})
	/usr/local/go/src/runtime/stack.go:627 +0x1ad fp=0x7fa6af8ea5f0 sp=0x7fa6af8ea590 pc=0x7fa6afdd306d
runtime.adjustframe(0x7fa6af8ea808, 0x7fa6af8ea6e8)
	/usr/local/go/src/runtime/stack.go:684 +0xdb fp=0x7fa6af8ea680 sp=0x7fa6af8ea5f0 pc=0x7fa6afdd319b
runtime.copystack(0xc0005da680, 0x7fa6af8eab20?)
	/usr/local/go/src/runtime/stack.go:935 +0x2c5 fp=0x7fa6af8ea978 sp=0x7fa6af8ea680 pc=0x7fa6afdd3945
runtime.shrinkstack(0xc0005da680)
	/usr/local/go/src/runtime/stack.go:1218 +0x113 fp=0x7fa6af8ea998 sp=0x7fa6af8ea978 pc=0x7fa6afdd4833
runtime.scanstack(0xc0005da680, 0x1a?)
	/usr/local/go/src/runtime/mgcmark.go:776 +0xdb fp=0x7fa6af8eacd0 sp=0x7fa6af8ea998 pc=0x7fa6afda2e9b
runtime.markroot.func1()
	/usr/local/go/src/runtime/mgcmark.go:240 +0xb5 fp=0x7fa6af8ead20 sp=0x7fa6af8eacd0 pc=0x7fa6afda1eb5
runtime.markroot(0xc00004f240, 0x77, 0x1)
	/usr/local/go/src/runtime/mgcmark.go:214 +0x1a8 fp=0x7fa6af8eadc8 sp=0x7fa6af8ead20 pc=0x7fa6afda1b48
runtime.gcDrain(0xc00004f240, 0x7)
	/usr/local/go/src/runtime/mgcmark.go:1069 +0x379 fp=0x7fa6af8eae28 sp=0x7fa6af8eadc8 pc=0x7fa6afda3a99
runtime.gcBgMarkWorker.func2()
	/usr/local/go/src/runtime/mgc.go:1387 +0x6f fp=0x7fa6af8eae78 sp=0x7fa6af8eae28 pc=0x7fa6afda01cf
runtime.systemstack()
	/usr/local/go/src/runtime/asm_amd64.s:509 +0x47 fp=0x7fa6af8eae88 sp=0x7fa6af8eae78 pc=0x7fa6afdeaca7

goroutine 5 [GC worker (active)]:
runtime.systemstack_switch()
	/usr/local/go/src/runtime/asm_amd64.s:474 +0x8 fp=0xc000062750 sp=0xc000062740 pc=0x7fa6afdeac48
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1355 +0x1f6 fp=0xc0000627e0 sp=0xc000062750 pc=0x7fa6afd9fe96
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000627e8 sp=0xc0000627e0 pc=0x7fa6afdecc41
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1219 +0x1c

goroutine 2846 [select]:
runtime.gopark(0xc000828f90?, 0x2?, 0xe0?, 0x8d?, 0xc000828f1c?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000828db8 sp=0xc000828d98 pc=0x7fa6afdbd58e
runtime.selectgo(0xc000828f90, 0xc000828f18, 0xc000126d80?, 0x0, 0xc000772090?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000828ed8 sp=0xc000828db8 pc=0x7fa6afdcd065
net/http.(*persistConn).writeLoop(0xc00073e240)
	/usr/local/go/src/net/http/transport.go:2421 +0xe5 fp=0xc000828fc8 sp=0xc000828ed8 pc=0x7fa6b0314065
net/http.(*Transport).dialConn.func6()
	/usr/local/go/src/net/http/transport.go:1777 +0x25 fp=0xc000828fe0 sp=0xc000828fc8 pc=0x7fa6b0310e65
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000828fe8 sp=0xc000828fe0 pc=0x7fa6afdecc41
created by net/http.(*Transport).dialConn in goroutine 2839
	/usr/local/go/src/net/http/transport.go:1777 +0x16f1

goroutine 17 [semacquire, locked to thread]:
runtime.gopark(0x60?, 0x7fa6c8e70890?, 0x60?, 0xc0?, 0x50?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000f71958 sp=0xc000f71938 pc=0x7fa6afdbd58e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.semacquire1(0xc00053ad90, 0x18?, 0x1, 0x0, 0x8b?)
	/usr/local/go/src/runtime/sema.go:160 +0x21c fp=0xc000f719c0 sp=0xc000f71958 pc=0x7fa6afdce09c
sync.runtime_Semacquire(0xc0010f0bb8?)
	/usr/local/go/src/runtime/sema.go:62 +0x25 fp=0xc000f719f8 sp=0xc000f719c0 pc=0x7fa6afde8e45
sync.(*WaitGroup).Wait(0x7fa6b1df2400?)
	/usr/local/go/src/sync/waitgroup.go:116 +0x48 fp=0xc000f71a20 sp=0xc000f719f8 pc=0x7fa6afdf4888
golang.org/x/sync/errgroup.(*Group).Wait(0xc00053ad80)
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:56 +0x25 fp=0xc000f71a40 sp=0xc000f71a20 pc=0x7fa6b047a085
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ingestStream(0xc000740c60, {0x7fa6b1ef04b8, 0xc00064a140})
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:285 +0x712 fp=0xc000f71c00 sp=0xc000f71a40 pc=0x7fa6b17ccc52
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).executeIngest(0xc000740c60, {0x7fa6b1ef04b8, 0xc00064a140})
	/adbc/go/adbc/driver/snowflake/statement.go:443 +0x7a fp=0xc000f71c60 sp=0xc000f71c00 pc=0x7fa6b17e92da
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ExecuteUpdate(0xc00119cf60?, {0x7fa6b1ef04b8?, 0xc00064a140?})
	/adbc/go/adbc/driver/snowflake/statement.go:504 +0x2b6 fp=0xc000f71d80 sp=0xc000f71c60 pc=0x7fa6b17e9b36
main.SnowflakeStatementExecuteQuery(0x7fa6afd89e12?, 0x0, 0x7ffc9cdfcd08, 0x7ffc9cdfcd40)
	/adbc/go/adbc/pkg/snowflake/driver.go:1525 +0x18a fp=0xc000f71e00 sp=0xc000f71d80 pc=0x7fa6b180b50a
_cgoexp_af66e644f38c_SnowflakeStatementExecuteQuery(0x7ffc9cdfcca0)
	_cgo_gotypes.go:1090 +0x2a fp=0xc000f71e30 sp=0xc000f71e00 pc=0x7fa6b180f08a
runtime.cgocallbackg1(0x7fa6b180f060, 0xc0011b5fe0?, 0x0)
	/usr/local/go/src/runtime/cgocall.go:399 +0x2c2 fp=0xc000f71f00 sp=0xc000f71e30 pc=0x7fa6afd882a2
runtime.cgocallbackg(0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/cgocall.go:315 +0x125 fp=0xc000f71f90 sp=0xc000f71f00 pc=0x7fa6afd87f45
runtime.cgocallbackg(0x7fa6b180f060, 0x7ffc9cdfcca0, 0x0)
	<autogenerated>:1 +0x2b fp=0xc000f71fb8 sp=0xc000f71f90 pc=0x7fa6afdef1cb
runtime.cgocallback(0x0, 0x0, 0x0)
	/usr/local/go/src/runtime/asm_amd64.s:1035 +0xcd fp=0xc000f71fe0 sp=0xc000f71fb8 pc=0x7fa6afdec9ed
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000f71fe8 sp=0xc000f71fe0 pc=0x7fa6afdecc41

goroutine 2 [force gc (idle), 2 minutes]:
runtime.gopark(0x5f0ada4d43901?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000060fa8 sp=0xc000060f88 pc=0x7fa6afdbd58e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:322 +0xb8 fp=0xc000060fe0 sp=0xc000060fa8 pc=0x7fa6afdbd418
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000060fe8 sp=0xc000060fe0 pc=0x7fa6afdecc41
created by runtime.init.6 in goroutine 1
	/usr/local/go/src/runtime/proc.go:310 +0x1a

goroutine 3 [GC sweep wait]:
runtime.gopark(0x7fa6b30baf01?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000061778 sp=0xc000061758 pc=0x7fa6afdbd58e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.bgsweep(0x0?)
	/usr/local/go/src/runtime/mgcsweep.go:321 +0xdf fp=0xc0000617c8 sp=0xc000061778 pc=0x7fa6afda90bf
runtime.gcenable.func1()
	/usr/local/go/src/runtime/mgc.go:200 +0x25 fp=0xc0000617e0 sp=0xc0000617c8 pc=0x7fa6afd9e1c5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000617e8 sp=0xc0000617e0 pc=0x7fa6afdecc41
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:200 +0x66

goroutine 4 [GC scavenge wait]:
runtime.gopark(0x103ae3?, 0x8d91a?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000061f70 sp=0xc000061f50 pc=0x7fa6afdbd58e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.(*scavengerState).park(0x7fa6b30bc000)
	/usr/local/go/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc000061fa0 sp=0xc000061f70 pc=0x7fa6afda6929
runtime.bgscavenge(0x0?)
	/usr/local/go/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc000061fc8 sp=0xc000061fa0 pc=0x7fa6afda6ed9
runtime.gcenable.func2()
	/usr/local/go/src/runtime/mgc.go:201 +0x25 fp=0xc000061fe0 sp=0xc000061fc8 pc=0x7fa6afd9e165
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000061fe8 sp=0xc000061fe0 pc=0x7fa6afdecc41
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:201 +0xa5

goroutine 18 [finalizer wait, 1439 minutes]:
runtime.gopark(0x0?, 0xc0008725e8?, 0x20?, 0xbd?, 0x1000000010?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc00081ae20 sp=0xc00081ae00 pc=0x7fa6afdbd58e
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc00081afe0 sp=0xc00081ae20 pc=0x7fa6afd9d1e7
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00081afe8 sp=0xc00081afe0 pc=0x7fa6afdecc41
created by runtime.createfing in goroutine 1
	/usr/local/go/src/runtime/mfinal.go:163 +0x3d

goroutine 19 [GC worker (idle)]:
runtime.gopark(0x5f0b8540fed82?, 0x3?, 0x6d?, 0x1?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc00005c750 sp=0xc00005c730 pc=0x7fa6afdbd58e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1295 +0xe5 fp=0xc00005c7e0 sp=0xc00005c750 pc=0x7fa6afd9fd85
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00005c7e8 sp=0xc00005c7e0 pc=0x7fa6afdecc41
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1219 +0x1c

goroutine 2860 [preempted (scan)]:
runtime.semrelease1(0x7fa6b30bd4d8?, 0x0, 0x0)
	/usr/local/go/src/runtime/sema.go:175 +0x1e9 fp=0xc00100d0c8 sp=0xc00100d0c0 pc=0x7fa6afdce3c9
runtime.semrelease(...)
	/usr/local/go/src/runtime/sema.go:172
runtime.gcStart({0xc0082b2000?, 0x294000?, 0x293d00?})
	/usr/local/go/src/runtime/mgc.go:775 +0x48d fp=0xc00100d160 sp=0xc00100d0c8 pc=0x7fa6afd9ea4d
runtime.mallocgc(0x293d00, 0x7fa6b1cc9880, 0x1)
	/usr/local/go/src/runtime/malloc.go:1246 +0x76f fp=0xc00100d1c8 sp=0xc00100d160 pc=0x7fa6afd90e2f
runtime.makeslice(0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/slice.go:103 +0x49 fp=0xc00100d1f0 sp=0xc00100d1c8 pc=0x7fa6afdd1a09
github.com/apache/arrow/go/v17/parquet/pqarrow.writeDenseArrow(0xc0006b65f0, {0x7fa6b1f19d30?, 0xc001111e00}, {0x7fa6b1f132f8, 0xc0000236d0?}, {0xc006bd6000, 0x1b7e0, 0x100000}, {0x0, 0x0, ...}, ...)
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/encode_arrow.go:486 +0x2225 fp=0xc00100d7d0 sp=0xc00100d1f0 pc=0x7fa6b0bfac05
github.com/apache/arrow/go/v17/parquet/pqarrow.WriteArrowToColumn({0x7fa6b1ef0480, 0xc00119c180}, {0x7fa6b1f19d30, 0xc001111e00}, {0x7fa6b1f132f8?, 0xc0000236d0?}, {0xc006bd6000, 0x1b7e0, 0x100000}, {0x0, ...}, ...)
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/encode_arrow.go:232 +0x32f fp=0xc00100d8a0 sp=0xc00100d7d0 pc=0x7fa6b0bf88af
github.com/apache/arrow/go/v17/parquet/pqarrow.(*arrowColumnWriter).Write(0xc00112fb08, {0x7fa6b1ef0480, 0xc00119c180})
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/encode_arrow.go:193 +0x4f3 fp=0xc00100dad0 sp=0xc00100d8a0 pc=0x7fa6b0bf8313
github.com/apache/arrow/go/v17/parquet/pqarrow.(*FileWriter).WriteColumnChunked(0xc00058e000, 0x7fa6b30ed020?, 0xc00112fbd8?, 0x1?)
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/file_writer.go:330 +0xe5 fp=0xc00100db88 sp=0xc00100dad0 pc=0x7fa6b0bffa05
github.com/apache/arrow/go/v17/parquet/pqarrow.(*FileWriter).WriteColumnData(0xc00058e000?, {0x7fa6b1f132f8, 0xc0006b63c0})
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/file_writer.go:339 +0xc9 fp=0xc00100dc10 sp=0xc00100db88 pc=0x7fa6b0bffb29
github.com/apache/arrow/go/v17/parquet/pqarrow.(*FileWriter).WriteBuffered(0xc00058e000, {0x7fa6b1f10ee0, 0xc00119c0f0})
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/file_writer.go:203 +0x3e5 fp=0xc00100dd90 sp=0xc00100dc10 pc=0x7fa6b0bff545
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.writeParquet(0xc0011a6100?, {0x7fa6b1ed2a88?, 0xc0005c4180}, 0x7fa6b0175f20?, 0xa00000, 0xc000102060?, {{0x7fa6b1ee63e0, 0xc0006ab1e8}, 0x0, 0x0, ...})
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:345 +0x1cb fp=0xc00100de78 sp=0xc00100dd90 pc=0x7fa6b17cdb4b
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.runParallelParquetWriters.func3()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:409 +0x185 fp=0xc00100df78 sp=0xc00100de78 pc=0x7fa6b17ce2a5
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56 fp=0xc00100dfe0 sp=0xc00100df78 pc=0x7fa6b047a1f6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00100dfe8 sp=0xc00100dfe0 pc=0x7fa6afdecc41
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 2868
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96

goroutine 2871 [select]:
runtime.gopark(0xc000dbc028?, 0x6?, 0xe8?, 0xbc?, 0xc000dbbe3c?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000dbbcc0 sp=0xc000dbbca0 pc=0x7fa6afdbd58e
runtime.selectgo(0xc000dbc028, 0xc000dbbe30, 0x7fa6b1820ce1?, 0x0, 0x7fa6b030c405?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000dbbde0 sp=0xc000dbbcc0 pc=0x7fa6afdcd065
net/http.(*persistConn).roundTrip(0xc00073e000, 0xc000126740)
	/usr/local/go/src/net/http/transport.go:2652 +0x979 fp=0xc000dbc098 sp=0xc000dbbde0 pc=0x7fa6b0315099
net/http.(*Transport).roundTrip(0x7fa6b2ff18a0, 0xc0011f8a00)
	/usr/local/go/src/net/http/transport.go:604 +0x79a fp=0xc000dbc2c0 sp=0xc000dbc098 pc=0x7fa6b030941a
net/http.(*Transport).RoundTrip(0xc0011f8a00?, 0x7fa6b1ed33c0?)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x13 fp=0xc000dbc2e0 sp=0xc000dbc2c0 pc=0x7fa6b02fe1b3
net/http.send(0xc0011f8900, {0x7fa6b1ed33c0, 0x7fa6b2ff18a0}, {0x1?, 0x7fa6afe10334?, 0x7fa6b30bbf00?})
	/usr/local/go/src/net/http/client.go:260 +0x606 fp=0xc000dbc510 sp=0xc000dbc2e0 pc=0x7fa6b02d3b66
net/http.(*Client).send(0xc001102300, 0xc0011f8900, {0x7fa6b0248d29?, 0xc000dbc608?, 0x7fa6b30bbf00?})
	/usr/local/go/src/net/http/client.go:181 +0x98 fp=0xc000dbc588 sp=0xc000dbc510 pc=0x7fa6b02d33f8
net/http.(*Client).do(0xc001102300, 0xc0011f8900)
	/usr/local/go/src/net/http/client.go:724 +0x912 fp=0xc000dbc798 sp=0xc000dbc588 pc=0x7fa6b02d56d2
net/http.(*Client).Do(0xc000dbca50?, 0xc001102270?)
	/usr/local/go/src/net/http/client.go:590 +0x13 fp=0xc000dbc7b8 sp=0xc000dbc798 pc=0x7fa6b02d4d93
github.com/snowflakedb/gosnowflake.(*retryHTTP).execute(0xc000dbcad0)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/retry.go:325 +0x44a fp=0xc000dbcac0 sp=0xc000dbc7b8 pc=0x7fa6b1791a6a
github.com/snowflakedb/gosnowflake.postRestful({0x7fa6b1ef0480?, 0xc0005c5cb0?}, 0x7fa6afd9ab28?, 0xc000dbcc38?, 0x7fa6afea0e4e?, {0xc00079a000, 0xe5, 0xf0}, 0x1?, {0x7fa6b1ed34a0, ...}, ...)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/restful.go:174 +0x174 fp=0xc000dbcb58 sp=0xc000dbcac0 pc=0x7fa6b178b814
github.com/snowflakedb/gosnowflake.postRestfulQueryHelper({0x7fa6b1ef0480, 0xc0005c5cb0}, 0xc000584270, 0xc00011e450, 0xc000dbce10?, {0xc00079a000, 0xe5, 0xf0}, 0xc000dbce01?, {0x73, ...}, ...)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/restful.go:246 +0x45e fp=0xc000dbcdb0 sp=0xc000dbcb58 pc=0x7fa6b178c17e
github.com/snowflakedb/gosnowflake.postRestfulQuery({0x7fa6b1ef0480, 0xc0005c5cb0}, 0xc000584270, 0x7fa6c9bc05b8?, 0xc000d6cf30?, {0xc00079a000?, 0xc0004fa310?, 0x70?}, 0x70?, {0x73, ...}, ...)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/restful.go:213 +0x5f fp=0xc000dbce40 sp=0xc000dbcdb0 pc=0x7fa6b178bbbf
github.com/snowflakedb/gosnowflake.(*snowflakeConn).exec(0xc000467020, {0x7fa6b1ef0480, 0xc0005c5cb0}, {0xc0007fe000, 0x43}, 0x0, 0x0, 0x0, {0x0, 0x0, ...})
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/connection.go:145 +0x85c fp=0xc000dbd0b0 sp=0xc000dbce40 pc=0x7fa6b174d8dc
github.com/snowflakedb/gosnowflake.(*snowflakeConn).ExecContext(0xc000467020, {0x7fa6b1ef0480, 0xc0005c5bf0}, {0xc0007fe000, 0x43}, {0x0, 0x0, 0x0})
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/connection.go:309 +0x225 fp=0xc000dbde70 sp=0xc000dbd0b0 pc=0x7fa6b174eb05
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.uploadStream({0x7fa6b1ef04b8, 0xc0009f62d0}, {0x7fa6c9252418, 0xc000467020}, {0x7fa6b1ed2c08?, 0xc0005c4270}, {0xc0010464b0?, 0xc00078b580?})
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:427 +0x126 fp=0xc000dbdee8 sp=0xc000dbde70 pc=0x7fa6b17ce626
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.uploadAllStreams.func1()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:464 +0x99 fp=0xc000dbdf78 sp=0xc000dbdee8 pc=0x7fa6b17cea39
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56 fp=0xc000dbdfe0 sp=0xc000dbdf78 pc=0x7fa6b047a1f6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000dbdfe8 sp=0xc000dbdfe0 pc=0x7fa6afdecc41
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 2870
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96

goroutine 2847 [IO wait]:
runtime.gopark(0x86d3103714bad6ff?, 0xb?, 0x0?, 0x0?, 0x3?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc0008255c8 sp=0xc0008255a8 pc=0x7fa6afdbd58e
runtime.netpollblock(0x7fa6afe02818?, 0xafd872c6?, 0xa6?)
	/usr/local/go/src/runtime/netpoll.go:564 +0xf7 fp=0xc000825600 sp=0xc0008255c8 pc=0x7fa6afdb5bf7
internal/poll.runtime_pollWait(0x7fa6c9a47558, 0x72)
	/usr/local/go/src/runtime/netpoll.go:343 +0x85 fp=0xc000825620 sp=0xc000825600 pc=0x7fa6afde7445
internal/poll.(*pollDesc).wait(0xc000462780?, 0xc000707800?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000825648 sp=0xc000825620 pc=0x7fa6afe5e8c7
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000462780, {0xc000707800, 0x1800, 0x1800})
	/usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a fp=0xc0008256e0 sp=0xc000825648 pc=0x7fa6afe5fbba
net.(*netFD).Read(0xc000462780, {0xc000707800?, 0xc00070780d?, 0x155?})
	/usr/local/go/src/net/fd_posix.go:55 +0x25 fp=0xc000825728 sp=0xc0008256e0 pc=0x7fa6b0221945
net.(*conn).Read(0xc0005a4040, {0xc000707800?, 0x17f3?, 0xc000707805?})
	/usr/local/go/src/net/net.go:179 +0x45 fp=0xc000825770 sp=0xc000825728 pc=0x7fa6b0232065
net.(*TCPConn).Read(0xc000825808?, {0xc000707800?, 0xc0010f0b28?, 0x18?})
	<autogenerated>:1 +0x25 fp=0xc0008257a0 sp=0xc000825770 pc=0x7fa6b0244ca5
crypto/tls.(*atLeastReader).Read(0xc0010f0b28, {0xc000707800?, 0xc0010f0b28?, 0x0?})
	/usr/local/go/src/crypto/tls/conn.go:805 +0x3b fp=0xc0008257e8 sp=0xc0008257a0 pc=0x7fa6b0283bbb
bytes.(*Buffer).ReadFrom(0xc0000e2d28, {0x7fa6b1ed5840, 0xc0010f0b28})
	/usr/local/go/src/bytes/buffer.go:211 +0x98 fp=0xc000825840 sp=0xc0008257e8 pc=0x7fa6afe829b8
crypto/tls.(*Conn).readFromUntil(0xc0000e2a80, {0x7fa6b1ed4a20?, 0xc0005a4040}, 0x1800?)
	/usr/local/go/src/crypto/tls/conn.go:827 +0xde fp=0xc000825880 sp=0xc000825840 pc=0x7fa6b0283d9e
crypto/tls.(*Conn).readRecordOrCCS(0xc0000e2a80, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:625 +0x250 fp=0xc000825c20 sp=0xc000825880 pc=0x7fa6b0281370
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc0000e2a80, {0xc001048000, 0x1000, 0x2c?})
	/usr/local/go/src/crypto/tls/conn.go:1369 +0x158 fp=0xc000825c90 sp=0xc000825c20 pc=0x7fa6b0287638
net/http.(*persistConn).Read(0xc00073e000, {0xc001048000?, 0xc000103260?, 0xc000825d38?})
	/usr/local/go/src/net/http/transport.go:1954 +0x4a fp=0xc000825cf0 sp=0xc000825c90 pc=0x7fa6b03118aa
bufio.(*Reader).fill(0xc00008cc60)
	/usr/local/go/src/bufio/bufio.go:113 +0x103 fp=0xc000825d28 sp=0xc000825cf0 pc=0x7fa6affdeb83
bufio.(*Reader).Peek(0xc00008cc60, 0x1)
	/usr/local/go/src/bufio/bufio.go:151 +0x53 fp=0xc000825d48 sp=0xc000825d28 pc=0x7fa6affdecb3
net/http.(*persistConn).readLoop(0xc00073e000)
	/usr/local/go/src/net/http/transport.go:2118 +0x1b9 fp=0xc000825fc8 sp=0xc000825d48 pc=0x7fa6b0312699
net/http.(*Transport).dialConn.func5()
	/usr/local/go/src/net/http/transport.go:1776 +0x25 fp=0xc000825fe0 sp=0xc000825fc8 pc=0x7fa6b0310ec5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000825fe8 sp=0xc000825fe0 pc=0x7fa6afdecc41
created by net/http.(*Transport).dialConn in goroutine 2826
	/usr/local/go/src/net/http/transport.go:1776 +0x169f

goroutine 2868 [semacquire]:
runtime.gopark(0xc000063550?, 0x7fa6afd90f54?, 0xe0?, 0xc1?, 0xc0000635b0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000063528 sp=0xc000063508 pc=0x7fa6afdbd58e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.semacquire1(0xc001044110, 0x18?, 0x1, 0x0, 0x8b?)
	/usr/local/go/src/runtime/sema.go:160 +0x21c fp=0xc000063590 sp=0xc000063528 pc=0x7fa6afdce09c
sync.runtime_Semacquire(0xc0008fe2e8?)
	/usr/local/go/src/runtime/sema.go:62 +0x25 fp=0xc0000635c8 sp=0xc000063590 pc=0x7fa6afde8e45
sync.(*WaitGroup).Wait(0x7fa6b1e5a4a0?)
	/usr/local/go/src/sync/waitgroup.go:116 +0x48 fp=0xc0000635f0 sp=0xc0000635c8 pc=0x7fa6afdf4888
golang.org/x/sync/errgroup.(*Group).Wait(0xc001044100)
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:56 +0x25 fp=0xc000063610 sp=0xc0000635f0 pc=0x7fa6b047a085
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.runParallelParquetWriters({0x7fa6b1ef04b8, 0xc00064aa50}, 0xc000466ea0, 0xa00000, 0xc0010b4160?, 0xc000716790, {{0x7fa6b1ee63e0, 0xc0006ab1e8}, 0x0, 0x0, ...}, ...)
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:392 +0x345 fp=0xc0000636b0 sp=0xc000063610 pc=0x7fa6b17ce065
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ingestStream.func4()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:257 +0x125 fp=0xc000063778 sp=0xc0000636b0 pc=0x7fa6b17ccfe5
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56 fp=0xc0000637e0 sp=0xc000063778 pc=0x7fa6b047a1f6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000637e8 sp=0xc0000637e0 pc=0x7fa6afdecc41
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 17
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96

goroutine 2848 [select]:
runtime.gopark(0xc000823f90?, 0x2?, 0xe0?, 0x3d?, 0xc000823f1c?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000823db8 sp=0xc000823d98 pc=0x7fa6afdbd58e
runtime.selectgo(0xc000823f90, 0xc000823f18, 0xc001044380?, 0x0, 0xc0011023f0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000823ed8 sp=0xc000823db8 pc=0x7fa6afdcd065
net/http.(*persistConn).writeLoop(0xc00073e000)
	/usr/local/go/src/net/http/transport.go:2421 +0xe5 fp=0xc000823fc8 sp=0xc000823ed8 pc=0x7fa6b0314065
net/http.(*Transport).dialConn.func6()
	/usr/local/go/src/net/http/transport.go:1777 +0x25 fp=0xc000823fe0 sp=0xc000823fc8 pc=0x7fa6b0310e65
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000823fe8 sp=0xc000823fe0 pc=0x7fa6afdecc41
created by net/http.(*Transport).dialConn in goroutine 2826
	/usr/local/go/src/net/http/transport.go:1777 +0x16f1

goroutine 2870 [chan receive]:
runtime.gopark(0x18?, 0x7fa6b1d2a5c0?, 0x0?, 0x0?, 0x7fa6ca2d8ad8?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc00081fdc0 sp=0xc00081fda0 pc=0x7fa6afdbd58e
runtime.chanrecv(0xc0004678c0, 0xc00081fec8, 0x1)
	/usr/local/go/src/runtime/chan.go:583 +0x3cd fp=0xc00081fe38 sp=0xc00081fdc0 pc=0x7fa6afd8a20d
runtime.chanrecv2(0xc000126640?, 0xc0009f6320?)
	/usr/local/go/src/runtime/chan.go:447 +0x12 fp=0xc00081fe60 sp=0xc00081fe38 pc=0x7fa6afd89e32
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.uploadAllStreams({0x7fa6b1ef04b8, 0xc00064aa50}, {0x7fa6c9252418?, 0xc000467020}, 0x0?, 0x7fa6b1aa70e0?, 0xc0009d7220, 0xc0010f0ba0)
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:448 +0x117 fp=0xc00081fef8 sp=0xc00081fe60 pc=0x7fa6b17ce7b7
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ingestStream.func5()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:281 +0xb3 fp=0xc00081ff78 sp=0xc00081fef8 pc=0x7fa6b17cce93
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56 fp=0xc00081ffe0 sp=0xc00081ff78 pc=0x7fa6b047a1f6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00081ffe8 sp=0xc00081ffe0 pc=0x7fa6afdecc41
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 17
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96

goroutine 2849 [select]:
runtime.gopark(0xc000063f88?, 0x2?, 0x0?, 0x0?, 0xc000063f84?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000063e30 sp=0xc000063e10 pc=0x7fa6afdbd58e
runtime.selectgo(0xc000063f88, 0xc000063f80, 0x0?, 0x0, 0x7fa6b1ef04b8?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000063f50 sp=0xc000063e30 pc=0x7fa6afdcd065
database/sql.(*DB).connectionOpener(0xc000117ba0, {0x7fa6b1ef04b8, 0xc0009f7220})
	/usr/local/go/src/database/sql/sql.go:1218 +0x87 fp=0xc000063fb8 sp=0xc000063f50 pc=0x7fa6b016ad67
database/sql.OpenDB.func1()
	/usr/local/go/src/database/sql/sql.go:791 +0x28 fp=0xc000063fe0 sp=0xc000063fb8 pc=0x7fa6b01693a8
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000063fe8 sp=0xc000063fe0 pc=0x7fa6afdecc41
created by database/sql.OpenDB in goroutine 17
	/usr/local/go/src/database/sql/sql.go:791 +0x165

goroutine 2845 [IO wait]:
runtime.gopark(0x1?, 0xb?, 0x0?, 0x0?, 0xa?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000827ab8 sp=0xc000827a98 pc=0x7fa6afdbd58e
runtime.netpollblock(0x7fa6afe02818?, 0xafd872c6?, 0xa6?)
	/usr/local/go/src/runtime/netpoll.go:564 +0xf7 fp=0xc000827af0 sp=0xc000827ab8 pc=0x7fa6afdb5bf7
internal/poll.runtime_pollWait(0x7fa6c9a47650, 0x72)
	/usr/local/go/src/runtime/netpoll.go:343 +0x85 fp=0xc000827b10 sp=0xc000827af0 pc=0x7fa6afde7445
internal/poll.(*pollDesc).wait(0xc000462a80?, 0xc000ffa000?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000827b38 sp=0xc000827b10 pc=0x7fa6afe5e8c7
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000462a80, {0xc000ffa000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a fp=0xc000827bd0 sp=0xc000827b38 pc=0x7fa6afe5fbba
net.(*netFD).Read(0xc000462a80, {0xc000ffa000?, 0x7fa6b030b5e0?, 0x7fa6b2ff1760?})
	/usr/local/go/src/net/fd_posix.go:55 +0x25 fp=0xc000827c18 sp=0xc000827bd0 pc=0x7fa6b0221945
net.(*conn).Read(0xc0005a4270, {0xc000ffa000?, 0x0?, 0xc000ba9520?})
	/usr/local/go/src/net/net.go:179 +0x45 fp=0xc000827c60 sp=0xc000827c18 pc=0x7fa6b0232065
net.(*TCPConn).Read(0x4?, {0xc000ffa000?, 0x7fa6afd895fa?, 0x0?})
	<autogenerated>:1 +0x25 fp=0xc000827c90 sp=0xc000827c60 pc=0x7fa6b0244ca5
net/http.(*persistConn).Read(0xc00073e240, {0xc000ffa000?, 0xc0005c17a0?, 0xc000827d38?})
	/usr/local/go/src/net/http/transport.go:1954 +0x4a fp=0xc000827cf0 sp=0xc000827c90 pc=0x7fa6b03118aa
bufio.(*Reader).fill(0xc0006bcea0)
	/usr/local/go/src/bufio/bufio.go:113 +0x103 fp=0xc000827d28 sp=0xc000827cf0 pc=0x7fa6affdeb83
bufio.(*Reader).Peek(0xc0006bcea0, 0x1)
	/usr/local/go/src/bufio/bufio.go:151 +0x53 fp=0xc000827d48 sp=0xc000827d28 pc=0x7fa6affdecb3
net/http.(*persistConn).readLoop(0xc00073e240)
	/usr/local/go/src/net/http/transport.go:2118 +0x1b9 fp=0xc000827fc8 sp=0xc000827d48 pc=0x7fa6b0312699
net/http.(*Transport).dialConn.func5()
	/usr/local/go/src/net/http/transport.go:1776 +0x25 fp=0xc000827fe0 sp=0xc000827fc8 pc=0x7fa6b0310ec5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000827fe8 sp=0xc000827fe0 pc=0x7fa6afdecc41
created by net/http.(*Transport).dialConn in goroutine 2839
	/usr/local/go/src/net/http/transport.go:1776 +0x169f

goroutine 2869 [select]:
runtime.gopark(0xc000820fa0?, 0x3?, 0x0?, 0x0?, 0xc000820f6a?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000820df8 sp=0xc000820dd8 pc=0x7fa6afdbd58e
runtime.selectgo(0xc000820fa0, 0xc000820f64, 0xc000820f98?, 0x0, 0x7fa6b2ff18a0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000820f18 sp=0xc000820df8 pc=0x7fa6afdcd065
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.runCopyTasks.func4()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:528 +0x105 fp=0xc000820fe0 sp=0xc000820f18 pc=0x7fa6b17cefc5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000820fe8 sp=0xc000820fe0 pc=0x7fa6afdecc41
created by github.com/apache/arrow-adbc/go/adbc/driver/snowflake.runCopyTasks in goroutine 17
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:524 +0x33d
Fatal Python error: Aborted

Thread 0x00007fa6e5e29b80 (most recent call first):
  File "/.venv/lib/python3.12/site-packages/adbc_driver_manager/dbapi.py", line 937 in adbc_ingest
  File "/.venv/lib/python3.12/site-packages/polars/dataframe/frame.py", line 3842 in write_database
  File "/home/appuser/app/db.py", line 19 in push_to_snowflake
  File "/home/appuser/app/main.py", line 21 in job
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 691 in run
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 173 in _run_job
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 101 in run_pending
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 854 in run_pending
  File "/home/appuser/app/main.py", line 39 in main
  File "/home/appuser/app/main.py", line 55 in <module>

Extension modules: adbc_driver_manager._lib, numpy._core._multiarray_umath, numpy._core._multiarray_tests, numpy.linalg._umath_linalg, pyarrow.lib, pyarrow._compute, pyarrow._acero, pyarrow._fs, pyarrow._csv, pyarrow._json, pyarrow._dataset, pyarrow._dataset_orc, pyarrow._parquet, pyarrow._parquet_encryption, pyarrow._dataset_parquet_encryption, pyarrow._dataset_parquet, adbc_driver_manager._reader, yaml._yaml (total: 18)

Component(s)

Go
@Niivii Niivii added the Type: bug Something isn't working label Aug 7, 2024
@Niivii Niivii changed the title Fatal Error in pqarrow.writeDenseArrow : invalid pointer [Go] Fatal Error in pqarrow.writeDenseArrow : invalid pointer Aug 7, 2024
@joellubi
Copy link
Member

joellubi commented Aug 7, 2024

@lidavidm Any potential insights related to this from your recent work cleaning up pointers with CGO?

@lidavidm
Copy link
Member

lidavidm commented Aug 7, 2024

I was looking at this, but the symptoms appear slightly different, so I wonder if it stems from the Parquet code somehow. Especially because this error explicitly blames a Parquet stackframe

@joellubi
Copy link
Member

joellubi commented Aug 7, 2024

In the stacktrace there's another goroutine that's preempted during a call to writeDenseArrow at line 486, which is a call to make(). These calls can trigger GC, which appears to be the case here. So that makes me wonder if it's just GC that's coming across an invalid pointer, and it's blaming writeDenseArrow because it was triggered by an allocation coming from that function.

Does this sound like it could be the case? There are some other explanations I'm exploring as well.

@lidavidm
Copy link
Member

lidavidm commented Aug 7, 2024

It says explicitly that the invalid pointer is in that particular stack frame, though. I don't think make would be prone to creating invalid pointers when that's a builtin (otherwise Go itself would be broken). I'd be more likely to guess that, for instance, we aren't properly using the mallocator in that function and so are violating CGo rules.

@zeroshade
Copy link
Member

How reproducible is this? If it's reliably reproducible then we should be able to debug it

@Niivii
Copy link
Author

Niivii commented Aug 8, 2024

Can't reproduce on my side, this code has been running for hundreds of iterations without issue.
The exact same data hasn't trigger the fatal error when re-run.

@lidavidm
Copy link
Member

lidavidm commented Aug 8, 2024

@zeroshade I assume we'll just have to audit the code here similarly to other cases like this.

@zeroshade
Copy link
Member

@lidavidm yup. sounds like it. If it was reliably reproducible then it wasn't likely to be the garbage data issue like previously so I was holding out hope. I'll try to find some time today / tomorrow to go through the code there and see if anything stands out to me.

It might be a good time to try to hook up using Pin and such instead of relying on malloc/calloc and the mallocator honestly.

@lidavidm
Copy link
Member

lidavidm commented Aug 8, 2024

Hmm, yeah, though we still get "exposed" due to the C Data Interface

@zeroshade
Copy link
Member

right, we could basically make it so the cdata package in Arrow proper just uses pinner.Pin on buffers when we export them, rather than relying on needing the mallocator etc.

@zeroshade
Copy link
Member

Hmm, I wonder how polars is constructing / passing the stream of batches from the table.... and whether there's something there that's the problem? though that still feels like it's likely something in our cdata import code path....

@lidavidm
Copy link
Member

Perhaps, but even then we should be guarding against things. I doubt Polars is deliberately inserting Go pointers into structures, so it could be a case of us needing to clear uninitialized memory again.

@joellubi
Copy link
Member

@Niivii Is there any more context you can provide about this example? Specifically it would be helpful to see how table is constructed, but anything else you think might be relevant would be appreciated.

@Niivii
Copy link
Author

Niivii commented Aug 13, 2024 •
edited
Loading

Usual workflow if that helps:

df.glimpse()
image

Snowflake flow:
image
image

Snowflake table DDL is all 23 columns as varchar
Source SQL Server table for the DataFrame is also a combination of different char sizes

This is executed at night so the warehouse isn't heavily loaded + as you can see the amount of records is really light.

@Niivii
Copy link
Author

Niivii commented Aug 26, 2024

Hello, another stacktrace of the issue happening again if that can help:

runtime: bad pointer in frame github.com/apache/arrow/go/v17/parquet/pqarrow.writeDenseArrow at 0xc0007175e0: 0x1
fatal error: invalid pointer found on stack

runtime stack:
runtime.throw({0x7fc9c53fc82f?, 0x7fc9c6976908?})
	/usr/local/go/src/runtime/panic.go:1077 +0x5e fp=0x7fc976afc568 sp=0x7fc976afc538 pc=0x7fc9c398169e
runtime.adjustpointers(0x7fc976afc7e0?, 0x7fc976afc628, 0x7fc9c39a4d65?, {0x7fc976afc7e0?, 0x0?})
	/usr/local/go/src/runtime/stack.go:627 +0x1ad fp=0x7fc976afc5c8 sp=0x7fc976afc568 pc=0x7fc9c399a06d
runtime.adjustframe(0x7fc976afc7e0, 0x7fc976afc6c0)
	/usr/local/go/src/runtime/stack.go:684 +0xdb fp=0x7fc976afc658 sp=0x7fc976afc5c8 pc=0x7fc9c399a19b
runtime.copystack(0xc0008ee1a0, 0x7fc976afcaf8?)
	/usr/local/go/src/runtime/stack.go:935 +0x2c5 fp=0x7fc976afc950 sp=0x7fc976afc658 pc=0x7fc9c399a945
runtime.shrinkstack(0xc0008ee1a0)
	/usr/local/go/src/runtime/stack.go:1218 +0x113 fp=0x7fc976afc970 sp=0x7fc976afc950 pc=0x7fc9c399b833
runtime.scanstack(0xc0008ee1a0, 0x4?)
	/usr/local/go/src/runtime/mgcmark.go:776 +0xdb fp=0x7fc976afcca8 sp=0x7fc976afc970 pc=0x7fc9c3969e9b
runtime.markroot.func1()
	/usr/local/go/src/runtime/mgcmark.go:240 +0xb5 fp=0x7fc976afccf8 sp=0x7fc976afcca8 pc=0x7fc9c3968eb5
runtime.markroot(0xc000053740, 0x3a, 0x1)
	/usr/local/go/src/runtime/mgcmark.go:214 +0x1a8 fp=0x7fc976afcda0 sp=0x7fc976afccf8 pc=0x7fc9c3968b48
runtime.gcDrain(0xc000053740, 0x7)
	/usr/local/go/src/runtime/mgcmark.go:1069 +0x379 fp=0x7fc976afce00 sp=0x7fc976afcda0 pc=0x7fc9c396aa99
runtime.gcBgMarkWorker.func2()
	/usr/local/go/src/runtime/mgc.go:1387 +0x6f fp=0x7fc976afce50 sp=0x7fc976afce00 pc=0x7fc9c39671cf
runtime.systemstack()
	/usr/local/go/src/runtime/asm_amd64.s:509 +0x47 fp=0x7fc976afce60 sp=0x7fc976afce50 pc=0x7fc9c39b1ca7

goroutine 6 [GC worker (active)]:
runtime.systemstack_switch()
	/usr/local/go/src/runtime/asm_amd64.s:474 +0x8 fp=0xc000064750 sp=0xc000064740 pc=0x7fc9c39b1c48
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1355 +0x1f6 fp=0xc0000647e0 sp=0xc000064750 pc=0x7fc9c3966e96
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000647e8 sp=0xc0000647e0 pc=0x7fc9c39b3c41
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1219 +0x1c

goroutine 17 [select, locked to thread]:
runtime.gopark(0xc00061bbf8?, 0x6?, 0xb8?, 0xb8?, 0xc00061ba0c?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc00061b890 sp=0xc00061b870 pc=0x7fc9c398458e
runtime.selectgo(0xc00061bbf8, 0xc00061ba00, 0x7fc9c53e7ce1?, 0x0, 0x7fc9c3ed3405?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc00061b9b0 sp=0xc00061b890 pc=0x7fc9c3994065
net/http.(*persistConn).roundTrip(0xc000b52240, 0xc00017eb00)
	/usr/local/go/src/net/http/transport.go:2652 +0x979 fp=0xc00061bc68 sp=0xc00061b9b0 pc=0x7fc9c3edc099
net/http.(*Transport).roundTrip(0x7fc9c6bb88a0, 0xc0000bb600)
	/usr/local/go/src/net/http/transport.go:604 +0x79a fp=0xc00061be90 sp=0xc00061bc68 pc=0x7fc9c3ed041a
net/http.(*Transport).RoundTrip(0xc0000bb600?, 0x7fc9c5a9a3c0?)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x13 fp=0xc00061beb0 sp=0xc00061be90 pc=0x7fc9c3ec51b3
net/http.send(0xc0000bb500, {0x7fc9c5a9a3c0, 0x7fc9c6bb88a0}, {0x1?, 0x7fc9c39d7334?, 0x7fc9c6c82f00?})
	/usr/local/go/src/net/http/client.go:260 +0x606 fp=0xc00061c0e0 sp=0xc00061beb0 pc=0x7fc9c3e9ab66
net/http.(*Client).send(0xc0006d1170, 0xc0000bb500, {0x7fc9c3e0fd29?, 0xc00061c1d8?, 0x7fc9c6c82f00?})
	/usr/local/go/src/net/http/client.go:181 +0x98 fp=0xc00061c158 sp=0xc00061c0e0 pc=0x7fc9c3e9a3f8
net/http.(*Client).do(0xc0006d1170, 0xc0000bb500)
	/usr/local/go/src/net/http/client.go:724 +0x912 fp=0xc00061c368 sp=0xc00061c158 pc=0x7fc9c3e9c6d2
net/http.(*Client).Do(0xc00061c620?, 0xc000a231a0?)
	/usr/local/go/src/net/http/client.go:590 +0x13 fp=0xc00061c388 sp=0xc00061c368 pc=0x7fc9c3e9bd93
github.com/snowflakedb/gosnowflake.(*retryHTTP).execute(0xc00061c6a0)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/retry.go:325 +0x44a fp=0xc00061c690 sp=0xc00061c388 pc=0x7fc9c5358a6a
github.com/snowflakedb/gosnowflake.postRestful({0x7fc9c5ab7480?, 0xc000a22cf0?}, 0x12?, 0xc00061c808?, 0x7fc9c3a67e4e?, {0xc0006de6c0, 0x11b, 0x120}, 0xc00061c7a8?, {0x7fc9c5a9a4a0, ...}, ...)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/restful.go:174 +0x174 fp=0xc00061c728 sp=0xc00061c690 pc=0x7fc9c5352814
github.com/snowflakedb/gosnowflake.postRestfulQueryHelper({0x7fc9c5ab7480, 0xc000a22cf0}, 0xc0006ccf70, 0xc000066438, 0xc00061c9e0?, {0xc0006de6c0, 0x11b, 0x120}, 0xc00061ca01?, {0xbe, ...}, ...)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/restful.go:246 +0x45e fp=0xc00061c980 sp=0xc00061c728 pc=0x7fc9c535317e
github.com/snowflakedb/gosnowflake.postRestfulQuery({0x7fc9c5ab7480, 0xc000a22cf0}, 0xc0006ccf70, 0x7fc9ca7355b8?, 0xc00061cb00?, {0xc0006de6c0?, 0xc0001568c0?, 0x70?}, 0x70?, {0xbe, ...}, ...)
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/restful.go:213 +0x5f fp=0xc00061ca10 sp=0xc00061c980 pc=0x7fc9c5352bbf
github.com/snowflakedb/gosnowflake.(*snowflakeConn).exec(0xc00070e060, {0x7fc9c5ab7480, 0xc000a22cf0}, {0x7fc9c5429c59, 0x79}, 0x0, 0x0, 0x0, {0x0, 0x0, ...})
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/connection.go:145 +0x85c fp=0xc00061cc80 sp=0xc00061ca10 pc=0x7fc9c53148dc
github.com/snowflakedb/gosnowflake.(*snowflakeConn).ExecContext(0xc00070e060, {0x7fc9c5ab74b8, 0xc000cf64b0}, {0x7fc9c5429c59, 0x79}, {0x0, 0x0, 0x0})
	/root/go/pkg/mod/github.com/snowflakedb/[email protected]/connection.go:309 +0x225 fp=0xc00061da40 sp=0xc00061cc80 pc=0x7fc9c5315b05
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ingestStream(0xc000504000, {0x7fc9c5ab74b8, 0xc000cf64b0})
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:271 +0x5e3 fp=0xc00061dc00 sp=0xc00061da40 pc=0x7fc9c5393b23
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).executeIngest(0xc000504000, {0x7fc9c5ab74b8, 0xc000cf64b0})
	/adbc/go/adbc/driver/snowflake/statement.go:443 +0x7a fp=0xc00061dc60 sp=0xc00061dc00 pc=0x7fc9c53b02da
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ExecuteUpdate(0xc00050b200?, {0x7fc9c5ab74b8?, 0xc000cf64b0?})
	/adbc/go/adbc/driver/snowflake/statement.go:504 +0x2b6 fp=0xc00061dd80 sp=0xc00061dc60 pc=0x7fc9c53b0b36
main.SnowflakeStatementExecuteQuery(0x7fc9c3950e12?, 0x0, 0x7fff06a61ab8, 0x7fff06a61af0)
	/adbc/go/adbc/pkg/snowflake/driver.go:1525 +0x18a fp=0xc00061de00 sp=0xc00061dd80 pc=0x7fc9c53d250a
_cgoexp_af66e644f38c_SnowflakeStatementExecuteQuery(0x7fff06a61a50)
	_cgo_gotypes.go:1090 +0x2a fp=0xc00061de30 sp=0xc00061de00 pc=0x7fc9c53d608a
runtime.cgocallbackg1(0x7fc9c53d6060, 0xc000605fe0?, 0x0)
	/usr/local/go/src/runtime/cgocall.go:399 +0x2c2 fp=0xc00061df00 sp=0xc00061de30 pc=0x7fc9c394f2a2
runtime.cgocallbackg(0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/cgocall.go:315 +0x125 fp=0xc00061df90 sp=0xc00061df00 pc=0x7fc9c394ef45
runtime.cgocallbackg(0x7fc9c53d6060, 0x7fff06a61a50, 0x0)
	<autogenerated>:1 +0x2b fp=0xc00061dfb8 sp=0xc00061df90 pc=0x7fc9c39b61cb
runtime.cgocallback(0x0, 0x0, 0x0)
	/usr/local/go/src/runtime/asm_amd64.s:1035 +0xcd fp=0xc00061dfe0 sp=0xc00061dfb8 pc=0x7fc9c39b39ed
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00061dfe8 sp=0xc00061dfe0 pc=0x7fc9c39b3c41

goroutine 2 [force gc (idle), 1 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000062fa8 sp=0xc000062f88 pc=0x7fc9c398458e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:322 +0xb8 fp=0xc000062fe0 sp=0xc000062fa8 pc=0x7fc9c3984418
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000062fe8 sp=0xc000062fe0 pc=0x7fc9c39b3c41
created by runtime.init.6 in goroutine 1
	/usr/local/go/src/runtime/proc.go:310 +0x1a

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000063778 sp=0xc000063758 pc=0x7fc9c398458e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.bgsweep(0x0?)
	/usr/local/go/src/runtime/mgcsweep.go:321 +0xdf fp=0xc0000637c8 sp=0xc000063778 pc=0x7fc9c39700bf
runtime.gcenable.func1()
	/usr/local/go/src/runtime/mgc.go:200 +0x25 fp=0xc0000637e0 sp=0xc0000637c8 pc=0x7fc9c39651c5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc0000637e8 sp=0xc0000637e0 pc=0x7fc9c39b3c41
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:200 +0x66

goroutine 4 [GC scavenge wait]:
runtime.gopark(0x3b9bcff7?, 0x3b9aca00?, 0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000063f70 sp=0xc000063f50 pc=0x7fc9c398458e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.(*scavengerState).park(0x7fc9c6c83000)
	/usr/local/go/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc000063fa0 sp=0xc000063f70 pc=0x7fc9c396d929
runtime.bgscavenge(0x0?)
	/usr/local/go/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc000063fc8 sp=0xc000063fa0 pc=0x7fc9c396ded9
runtime.gcenable.func2()
	/usr/local/go/src/runtime/mgc.go:201 +0x25 fp=0xc000063fe0 sp=0xc000063fc8 pc=0x7fc9c3965165
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000063fe8 sp=0xc000063fe0 pc=0x7fc9c39b3c41
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:201 +0xa5

goroutine 5 [finalizer wait]:
runtime.gopark(0x0?, 0xc00072f050?, 0xa0?, 0xa0?, 0x1000000010?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc00076ee20 sp=0xc00076ee00 pc=0x7fc9c398458e
runtime.runfinq()
	/usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc00076efe0 sp=0xc00076ee20 pc=0x7fc9c39641e7
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc00076efe8 sp=0xc00076efe0 pc=0x7fc9c39b3c41
created by runtime.createfing in goroutine 1
	/usr/local/go/src/runtime/mfinal.go:163 +0x3d

goroutine 7 [GC worker (idle)]:
runtime.gopark(0x7fc9c6cb5e80?, 0x2?, 0x9e?, 0x1e?, 0x0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000064f50 sp=0xc000064f30 pc=0x7fc9c398458e
runtime.gcBgMarkWorker()
	/usr/local/go/src/runtime/mgc.go:1295 +0xe5 fp=0xc000064fe0 sp=0xc000064f50 pc=0x7fc9c3966d85
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000064fe8 sp=0xc000064fe0 pc=0x7fc9c39b3c41
created by runtime.gcBgMarkStartWorkers in goroutine 1
	/usr/local/go/src/runtime/mgc.go:1219 +0x1c

goroutine 61 [select]:
runtime.gopark(0xc000079f90?, 0x2?, 0xe0?, 0x9d?, 0xc000079f1c?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000079db8 sp=0xc000079d98 pc=0x7fc9c398458e
runtime.selectgo(0xc000079f90, 0xc000079f18, 0xc000130640?, 0x0, 0xc000a23290?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000079ed8 sp=0xc000079db8 pc=0x7fc9c3994065
net/http.(*persistConn).writeLoop(0xc000b52240)
	/usr/local/go/src/net/http/transport.go:2421 +0xe5 fp=0xc000079fc8 sp=0xc000079ed8 pc=0x7fc9c3edb065
net/http.(*Transport).dialConn.func6()
	/usr/local/go/src/net/http/transport.go:1777 +0x25 fp=0xc000079fe0 sp=0xc000079fc8 pc=0x7fc9c3ed7e65
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000079fe8 sp=0xc000079fe0 pc=0x7fc9c39b3c41
created by net/http.(*Transport).dialConn in goroutine 54
	/usr/local/go/src/net/http/transport.go:1777 +0x16f1

goroutine 1073 [semacquire]:
runtime.gopark(0x7fc9c398458e?, 0x7fc9744e9290?, 0xc0?, 0x7b?, 0xc000077db0?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000077d28 sp=0xc000077d08 pc=0x7fc9c398458e
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:404
runtime.semacquire1(0xc00017eb50, 0x18?, 0x1, 0x0, 0x8b?)
	/usr/local/go/src/runtime/sema.go:160 +0x21c fp=0xc000077d90 sp=0xc000077d28 pc=0x7fc9c399509c
sync.runtime_Semacquire(0xc000b040a8?)
	/usr/local/go/src/runtime/sema.go:62 +0x25 fp=0xc000077dc8 sp=0xc000077d90 pc=0x7fc9c39afe45
sync.(*WaitGroup).Wait(0x7fc9c5a214a0?)
	/usr/local/go/src/sync/waitgroup.go:116 +0x48 fp=0xc000077df0 sp=0xc000077dc8 pc=0x7fc9c39bb888
golang.org/x/sync/errgroup.(*Group).Wait(0xc00017eb40)
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:56 +0x25 fp=0xc000077e10 sp=0xc000077df0 pc=0x7fc9c4041085
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.runParallelParquetWriters({0x7fc9c5ab74b8, 0xc000cf6be0}, 0xc00095a000, 0xa00000, 0xc0009b2200?, 0xc0004f0210, {{0x7fc9c5aad3e0, 0xc0006ba9f8}, 0x0, 0x0, ...}, ...)
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:392 +0x345 fp=0xc000077eb0 sp=0xc000077e10 pc=0x7fc9c5395065
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.(*statement).ingestStream.func4()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:257 +0x125 fp=0xc000077f78 sp=0xc000077eb0 pc=0x7fc9c5393fe5
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56 fp=0xc000077fe0 sp=0xc000077f78 pc=0x7fc9c40411f6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000077fe8 sp=0xc000077fe0 pc=0x7fc9c39b3c41
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 17
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96

goroutine 60 [IO wait]:
runtime.gopark(0xecee7dc0e3e227f4?, 0xb?, 0x0?, 0x0?, 0x3?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc0000785c8 sp=0xc0000785a8 pc=0x7fc9c398458e
runtime.netpollblock(0x7fc9c39c9818?, 0xc394e2c6?, 0xc9?)
	/usr/local/go/src/runtime/netpoll.go:564 +0xf7 fp=0xc000078600 sp=0xc0000785c8 pc=0x7fc9c397cbf7
internal/poll.runtime_pollWait(0x7fc9c00636d0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:343 +0x85 fp=0xc000078620 sp=0xc000078600 pc=0x7fc9c39ae445
internal/poll.(*pollDesc).wait(0xc00044ae80?, 0xc00084c000?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000078648 sp=0xc000078620 pc=0x7fc9c3a258c7
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00044ae80, {0xc00084c000, 0x8000, 0x8000})
	/usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a fp=0xc0000786e0 sp=0xc000078648 pc=0x7fc9c3a26bba
net.(*netFD).Read(0xc00044ae80, {0xc00084c000?, 0xc00084c00d?, 0x4cf?})
	/usr/local/go/src/net/fd_posix.go:55 +0x25 fp=0xc000078728 sp=0xc0000786e0 pc=0x7fc9c3de8945
net.(*conn).Read(0xc000160790, {0xc00084c000?, 0x7ff3?, 0xc00084c005?})
	/usr/local/go/src/net/net.go:179 +0x45 fp=0xc000078770 sp=0xc000078728 pc=0x7fc9c3df9065
net.(*TCPConn).Read(0xc000078808?, {0xc00084c000?, 0xc0007ec390?, 0x18?})
	<autogenerated>:1 +0x25 fp=0xc0000787a0 sp=0xc000078770 pc=0x7fc9c3e0bca5
crypto/tls.(*atLeastReader).Read(0xc0007ec390, {0xc00084c000?, 0xc0007ec390?, 0x0?})
	/usr/local/go/src/crypto/tls/conn.go:805 +0x3b fp=0xc0000787e8 sp=0xc0000787a0 pc=0x7fc9c3e4abbb
bytes.(*Buffer).ReadFrom(0xc0001290a8, {0x7fc9c5a9c840, 0xc0007ec390})
	/usr/local/go/src/bytes/buffer.go:211 +0x98 fp=0xc000078840 sp=0xc0000787e8 pc=0x7fc9c3a499b8
crypto/tls.(*Conn).readFromUntil(0xc000128e00, {0x7fc9c5a9ba20?, 0xc000160790}, 0x8000?)
	/usr/local/go/src/crypto/tls/conn.go:827 +0xde fp=0xc000078880 sp=0xc000078840 pc=0x7fc9c3e4ad9e
crypto/tls.(*Conn).readRecordOrCCS(0xc000128e00, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:625 +0x250 fp=0xc000078c20 sp=0xc000078880 pc=0x7fc9c3e48370
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:587
crypto/tls.(*Conn).Read(0xc000128e00, {0xc000729000, 0x1000, 0x2c?})
	/usr/local/go/src/crypto/tls/conn.go:1369 +0x158 fp=0xc000078c90 sp=0xc000078c20 pc=0x7fc9c3e4e638
net/http.(*persistConn).Read(0xc000b52240, {0xc000729000?, 0xc0004f2e40?, 0xc000078d38?})
	/usr/local/go/src/net/http/transport.go:1954 +0x4a fp=0xc000078cf0 sp=0xc000078c90 pc=0x7fc9c3ed88aa
bufio.(*Reader).fill(0xc0009796e0)
	/usr/local/go/src/bufio/bufio.go:113 +0x103 fp=0xc000078d28 sp=0xc000078cf0 pc=0x7fc9c3ba5b83
bufio.(*Reader).Peek(0xc0009796e0, 0x1)
	/usr/local/go/src/bufio/bufio.go:151 +0x53 fp=0xc000078d48 sp=0xc000078d28 pc=0x7fc9c3ba5cb3
net/http.(*persistConn).readLoop(0xc000b52240)
	/usr/local/go/src/net/http/transport.go:2118 +0x1b9 fp=0xc000078fc8 sp=0xc000078d48 pc=0x7fc9c3ed9699
net/http.(*Transport).dialConn.func5()
	/usr/local/go/src/net/http/transport.go:1776 +0x25 fp=0xc000078fe0 sp=0xc000078fc8 pc=0x7fc9c3ed7ec5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000078fe8 sp=0xc000078fe0 pc=0x7fc9c39b3c41
created by net/http.(*Transport).dialConn in goroutine 54
	/usr/local/go/src/net/http/transport.go:1776 +0x169f

goroutine 39 [IO wait, 1 minutes]:
runtime.gopark(0x1?, 0xb?, 0x0?, 0x0?, 0x5?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000a32ab8 sp=0xc000a32a98 pc=0x7fc9c398458e
runtime.netpollblock(0x7fc9c39c9818?, 0xc394e2c6?, 0xc9?)
	/usr/local/go/src/runtime/netpoll.go:564 +0xf7 fp=0xc000a32af0 sp=0xc000a32ab8 pc=0x7fc9c397cbf7
internal/poll.runtime_pollWait(0x7fc9c00634e0, 0x72)
	/usr/local/go/src/runtime/netpoll.go:343 +0x85 fp=0xc000a32b10 sp=0xc000a32af0 pc=0x7fc9c39ae445
internal/poll.(*pollDesc).wait(0xc00044a700?, 0xc0009ed000?, 0x0)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000a32b38 sp=0xc000a32b10 pc=0x7fc9c3a258c7
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00044a700, {0xc0009ed000, 0x1000, 0x1000})
	/usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a fp=0xc000a32bd0 sp=0xc000a32b38 pc=0x7fc9c3a26bba
net.(*netFD).Read(0xc00044a700, {0xc0009ed000?, 0x7fc9c3ed25e0?, 0x7fc9c6bb8760?})
	/usr/local/go/src/net/fd_posix.go:55 +0x25 fp=0xc000a32c18 sp=0xc000a32bd0 pc=0x7fc9c3de8945
net.(*conn).Read(0xc000160120, {0xc0009ed000?, 0x0?, 0xc00044d860?})
	/usr/local/go/src/net/net.go:179 +0x45 fp=0xc000a32c60 sp=0xc000a32c18 pc=0x7fc9c3df9065
net.(*TCPConn).Read(0x4?, {0xc0009ed000?, 0x7fc9c39505fa?, 0x0?})
	<autogenerated>:1 +0x25 fp=0xc000a32c90 sp=0xc000a32c60 pc=0x7fc9c3e0bca5
net/http.(*persistConn).Read(0xc0004c4240, {0xc0009ed000?, 0xc000103860?, 0xc000a32d38?})
	/usr/local/go/src/net/http/transport.go:1954 +0x4a fp=0xc000a32cf0 sp=0xc000a32c90 pc=0x7fc9c3ed88aa
bufio.(*Reader).fill(0xc000978b40)
	/usr/local/go/src/bufio/bufio.go:113 +0x103 fp=0xc000a32d28 sp=0xc000a32cf0 pc=0x7fc9c3ba5b83
bufio.(*Reader).Peek(0xc000978b40, 0x1)
	/usr/local/go/src/bufio/bufio.go:151 +0x53 fp=0xc000a32d48 sp=0xc000a32d28 pc=0x7fc9c3ba5cb3
net/http.(*persistConn).readLoop(0xc0004c4240)
	/usr/local/go/src/net/http/transport.go:2118 +0x1b9 fp=0xc000a32fc8 sp=0xc000a32d48 pc=0x7fc9c3ed9699
net/http.(*Transport).dialConn.func5()
	/usr/local/go/src/net/http/transport.go:1776 +0x25 fp=0xc000a32fe0 sp=0xc000a32fc8 pc=0x7fc9c3ed7ec5
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a32fe8 sp=0xc000a32fe0 pc=0x7fc9c39b3c41
created by net/http.(*Transport).dialConn in goroutine 35
	/usr/local/go/src/net/http/transport.go:1776 +0x169f

goroutine 40 [select, 1 minutes]:
runtime.gopark(0xc000076f90?, 0x2?, 0xe0?, 0x6d?, 0xc000076f1c?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000076db8 sp=0xc000076d98 pc=0x7fc9c398458e
runtime.selectgo(0xc000076f90, 0xc000076f18, 0xc0004cd9c0?, 0x0, 0xc00043c090?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000076ed8 sp=0xc000076db8 pc=0x7fc9c3994065
net/http.(*persistConn).writeLoop(0xc0004c4240)
	/usr/local/go/src/net/http/transport.go:2421 +0xe5 fp=0xc000076fc8 sp=0xc000076ed8 pc=0x7fc9c3edb065
net/http.(*Transport).dialConn.func6()
	/usr/local/go/src/net/http/transport.go:1777 +0x25 fp=0xc000076fe0 sp=0xc000076fc8 pc=0x7fc9c3ed7e65
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000076fe8 sp=0xc000076fe0 pc=0x7fc9c39b3c41
created by net/http.(*Transport).dialConn in goroutine 35
	/usr/local/go/src/net/http/transport.go:1777 +0x16f1

goroutine 53 [select, 1 minutes]:
runtime.gopark(0xc000a30f88?, 0x2?, 0x0?, 0x0?, 0xc000a30f84?)
	/usr/local/go/src/runtime/proc.go:398 +0xce fp=0xc000a30e30 sp=0xc000a30e10 pc=0x7fc9c398458e
runtime.selectgo(0xc000a30f88, 0xc000a30f80, 0x0?, 0x0, 0x0?, 0x1)
	/usr/local/go/src/runtime/select.go:327 +0x725 fp=0xc000a30f50 sp=0xc000a30e30 pc=0x7fc9c3994065
database/sql.(*DB).connectionOpener(0xc0006cde10, {0x7fc9c5ab74b8, 0xc000a180a0})
	/usr/local/go/src/database/sql/sql.go:1218 +0x87 fp=0xc000a30fb8 sp=0xc000a30f50 pc=0x7fc9c3d31d67
database/sql.OpenDB.func1()
	/usr/local/go/src/database/sql/sql.go:791 +0x28 fp=0xc000a30fe0 sp=0xc000a30fb8 pc=0x7fc9c3d303a8
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000a30fe8 sp=0xc000a30fe0 pc=0x7fc9c39b3c41
created by database/sql.OpenDB in goroutine 17
	/usr/local/go/src/database/sql/sql.go:791 +0x165

goroutine 1074 [preempted (scan)]:
runtime.semrelease1(0x7fc9c6c844d8?, 0x0, 0x0)
	/usr/local/go/src/runtime/sema.go:175 +0x1e9 fp=0xc0007170c8 sp=0xc0007170c0 pc=0x7fc9c39953c9
runtime.semrelease(...)
	/usr/local/go/src/runtime/sema.go:172
runtime.gcStart({0xc006584000?, 0xb18000?, 0xb178c0?})
	/usr/local/go/src/runtime/mgc.go:775 +0x48d fp=0xc000717160 sp=0xc0007170c8 pc=0x7fc9c3965a4d
runtime.mallocgc(0xb178c0, 0x7fc9c5890880, 0x1)
	/usr/local/go/src/runtime/malloc.go:1246 +0x76f fp=0xc0007171c8 sp=0xc000717160 pc=0x7fc9c3957e2f
runtime.makeslice(0x0?, 0x0?, 0x0?)
	/usr/local/go/src/runtime/slice.go:103 +0x49 fp=0xc0007171f0 sp=0xc0007171c8 pc=0x7fc9c3998a09
github.com/apache/arrow/go/v17/parquet/pqarrow.writeDenseArrow(0xc00056eaa0, {0x7fc9c5ae0d30?, 0xc000002600}, {0x7fc9c5ada2f8, 0xc000a9e870?}, {0xc005212000, 0x76508, 0x100000}, {0x0, 0x0, ...}, ...)
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/encode_arrow.go:486 +0x2225 fp=0xc0007177d0 sp=0xc0007171f0 pc=0x7fc9c47c1c05
github.com/apache/arrow/go/v17/parquet/pqarrow.WriteArrowToColumn({0x7fc9c5ab7480, 0xc000a221e0}, {0x7fc9c5ae0d30, 0xc000002600}, {0x7fc9c5ada2f8?, 0xc000a9e870?}, {0xc005212000, 0x76508, 0x100000}, {0x0, ...}, ...)
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/encode_arrow.go:232 +0x32f fp=0xc0007178a0 sp=0xc0007177d0 pc=0x7fc9c47bf8af
github.com/apache/arrow/go/v17/parquet/pqarrow.(*arrowColumnWriter).Write(0xc0008e3b08, {0x7fc9c5ab7480, 0xc000a221e0})
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/encode_arrow.go:193 +0x4f3 fp=0xc000717ad0 sp=0xc0007178a0 pc=0x7fc9c47bf313
github.com/apache/arrow/go/v17/parquet/pqarrow.(*FileWriter).WriteColumnChunked(0xc000156000, 0x7fc9c6cb4020?, 0xc0008e3bd8?, 0x1?)
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/file_writer.go:330 +0xe5 fp=0xc000717b88 sp=0xc000717ad0 pc=0x7fc9c47c6a05
github.com/apache/arrow/go/v17/parquet/pqarrow.(*FileWriter).WriteColumnData(0xc000156000?, {0x7fc9c5ada2f8, 0xc000cf7db0})
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/file_writer.go:339 +0xc9 fp=0xc000717c10 sp=0xc000717b88 pc=0x7fc9c47c6b29
github.com/apache/arrow/go/v17/parquet/pqarrow.(*FileWriter).WriteBuffered(0xc000156000, {0x7fc9c5ad7ee0, 0xc000a232f0})
	/root/go/pkg/mod/github.com/apache/arrow/go/[email protected]/parquet/pqarrow/file_writer.go:203 +0x3e5 fp=0xc000717d90 sp=0xc000717c10 pc=0x7fc9c47c6545
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.writeParquet(0xc000060f00?, {0x7fc9c5a99a88?, 0xc000a22000}, 0xc0006ba9f8?, 0xa00000, 0x0?, {{0x7fc9c5aad3e0, 0xc0006ba9f8}, 0x0, 0x0, ...})
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:345 +0x1cb fp=0xc000717e78 sp=0xc000717d90 pc=0x7fc9c5394b4b
github.com/apache/arrow-adbc/go/adbc/driver/snowflake.runParallelParquetWriters.func3()
	/adbc/go/adbc/driver/snowflake/bulk_ingestion.go:409 +0x185 fp=0xc000717f78 sp=0xc000717e78 pc=0x7fc9c53952a5
golang.org/x/sync/errgroup.(*Group).Go.func1()
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:78 +0x56 fp=0xc000717fe0 sp=0xc000717f78 pc=0x7fc9c40411f6
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:1650 +0x1 fp=0xc000717fe8 sp=0xc000717fe0 pc=0x7fc9c39b3c41
created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 1073
	/root/go/pkg/mod/golang.org/x/[email protected]/errgroup/errgroup.go:75 +0x96
Fatal Python error: Aborted

Thread 0x00007fc9e58d3b80 (most recent call first):
  File "/.venv/lib/python3.12/site-packages/adbc_driver_manager/dbapi.py", line 937 in adbc_ingest
  File "/.venv/lib/python3.12/site-packages/polars/dataframe/frame.py", line 3817 in write_database
  File "/home/appuser/app/db.py", line 76 in push_inserts_to_snowflake
  File "/home/appuser/app/main.py", line 103 in job
  File "/.venv/lib/python3.12/site-packages/tenacity/__init__.py", line 478 in __call__
  File "/.venv/lib/python3.12/site-packages/tenacity/__init__.py", line 336 in wrapped_f
  File "/home/appuser/app/main.py", line 38 in job_monthly
  File "/home/appuser/app/main.py", line 231 in job_monthly_once
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 691 in run
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 173 in _run_job
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 101 in run_pending
  File "/.venv/lib/python3.12/site-packages/schedule/__init__.py", line 854 in run_pending
  File "/home/appuser/app/main.py", line 264 in main
  File "/home/appuser/app/main.py", line 285 in <module>

Extension modules: yaml._yaml, adbc_driver_manager._lib, numpy._core._multiarray_umath, numpy.linalg._umath_linalg, pyarrow.lib, pyarrow._compute, pyarrow._acero, pyarrow._fs, pyarrow._csv, pyarrow._json, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pandas._libs.ops, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, pyarrow._dataset, pyarrow._dataset_orc, pyarrow._parquet, pyarrow._parquet_encryption, pyarrow._dataset_parquet_encryption, pyarrow._dataset_parquet, adbc_driver_manager._reader, sqlalchemy.cyextension.collections, sqlalchemy.cyextension.immutabledict, sqlalchemy.cyextension.processors, sqlalchemy.cyextension.resultproxy, sqlalchemy.cyextension.util, greenlet._greenlet, pyodbc (total: 73)

@assignUser assignUser transferred this issue from apache/arrow Aug 30, 2024
@zeroshade
Copy link
Member

@Niivii is this still reproducible? If so, could you provide a reproducer for us that we can look into it further?

@Niivii
Copy link
Author

Niivii commented Nov 8, 2024

I will bump our dataflow to the latest release and adbc driver 15 when it's out and let you know.
I will come back to you by the end of the month.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
Type: bug Something isn't working
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
4 participants
@lidavidm @zeroshade @Niivii @joellubi