feat(sourcemaps): Implement new tables supporting debug ids

[iambriccardo]

This PR aims at implementing the new tables used to support debug ids for source maps.

[jjbayer]

Suggested change

	bundle_id = models.CharField(max_length=32, null=True)
	bundle_id = models.UUIDField(null=True)

[jjbayer]

Couldn't this also be an integer field, like dist_id?

If we're going with strings, use the same maxlength as on Release:

sentry/src/sentry/models/release.py

Line 478 in 7c58e7c

version = models.CharField(max_length=DB_VERSION_LENGTH)

[iambriccardo]

Thanks! I was searching for the name of the release in the model and didn't think it could have been named version 😅.

[mitsuhiko]

We cannot guarantee that a release exists yet. I think since we likely have one artifact per release, a string is fine. Alternatively one could deduplicate through another table but that seems pointless.

[iambriccardo]

I will add @beezz to check if the tables are fine.

[jjbayer]

nit: I would drop the Index part of these intermediate tables, see for example

sentry/src/sentry/models/projectteam.py

Lines 43 to 47 in 7c58e7c

	class ProjectTeam(Model):
	__include_in_export__ = True
	project = FlexibleForeignKey("sentry.Project")
	team = FlexibleForeignKey("sentry.Team")

[iambriccardo]

I have followed the FileBlobIndex pattern but I agree, it's not an index. It's its own table.

[iambriccardo]

@mitsuhiko we were discussing with @beezz about trying to use a hash index on the debug_id, we are not sure about the performance though, because hash indexes on Postgres seem to be implemented with buckets that will require linear scan inside. On the Postgres website it is specified that the planner would try to use them for equality comparisons which is what we want for debug_id so maybe it may be worth trying.

[beezz]

Please add the tables to the database router before merging so they are created on the files cluster since we have FK to the sentry.File table.

[beezz]

You can remove this index since this set of columns is already in unique_together, which creates an index implicitly to enforce the uniqueness.

[beezz]

Is there a requirement for having this as a separate uuid field? Is the implicit primary key id field not enough?

[github-actions]

This PR has a migration; here is the generated SQL for src/sentry/migrations/0361_debug_id_artifact_bundle.py ()

--
-- Create model ArtifactBundle
--
CREATE TABLE "sentry_artifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "bundle_id" uuid NULL, "artifact_count" integer NOT NULL CHECK ("artifact_count" >= 0), "file_id" bigint NOT NULL);
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id" FOREIGN KEY ("file_id") REFERENCES "sentry_file" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_artifactbundle" VALIDATE CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" ON "sentry_artifactbundle" ("organization_id", "bundle_id");
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" UNIQUE USING INDEX "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq";
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_b5a13770" ON "sentry_artifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_file_id_1d6752c9" ON "sentry_artifactbundle" ("file_id");
--
-- Create model ReleaseArtifactBundle
--
CREATE TABLE "sentry_releaseartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "release_name" varchar(250) NOT NULL, "dist_id" bigint NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_releaseartifactbundle" VALIDATE CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" ON "sentry_releaseartifactbundle" ("organization_id", "release_name", "dist_id", "artifact_bundle_id");
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" UNIQUE USING INDEX "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq";
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_organization_id_10fc7929" ON "sentry_releaseartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_artifact_bundle_id_e0298a12" ON "sentry_releaseartifactbundle" ("artifact_bundle_id");
--
-- Create model ProjectArtifactBundle
--
CREATE TABLE "sentry_projectartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "project_id" bigint NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_projectartifactbundle" VALIDATE CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" ON "sentry_projectartifactbundle" ("project_id", "artifact_bundle_id");
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" UNIQUE USING INDEX "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq";
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_project_id_7588084b" ON "sentry_projectartifactbundle" ("project_id");
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_artifact_bundle_id_c3f68015" ON "sentry_projectartifactbundle" ("artifact_bundle_id");
--
-- Create model DebugIdArtifactBundle
--
CREATE TABLE "sentry_debugidartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "debug_id" uuid NOT NULL, "source_file_type" integer NOT NULL, "date_added" timestamp with time zone NOT NULL, "date_last_accessed" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_debugidartifactbundle" VALIDATE CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" ON "sentry_debugidartifactbundle" ("debug_id", "artifact_bundle_id", "source_file_type");
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" UNIQUE USING INDEX "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq";
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_organization_id_4db980bf" ON "sentry_debugidartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_artifact_bundle_id_91f3140f" ON "sentry_debugidartifactbundle" ("artifact_bundle_id");

[markstory]

Please add the tables to the database router before merging so they are created on the files cluster since we have FK to the sentry.File table.

Related to this - the files DB cluster does not automatically run migrations and you'll need to have those migrations deployed manually, or temporarily update the files DB configuration to run migrations.

[iambriccardo]

@markstory I had already a PR open for that, it is in draft because we first wanted to clarify the required tables.

It is available here: https://github.com/getsentry/getsentry/pull/9593

@beezz and I will sync on how we want to roll this out properly.

[mitsuhiko]

@asottile-sentry do we still need this? I really want to at least avoid new models to end up in here.

[asottile-sentry]

I don't think so -- I think this part of the patch can be discarded

[wedamija]

Should this be release_id and related to the Release table?

[iambriccardo]

@mitsuhiko wanted a weak reference to releases and decided to use the release.version.

[github-actions]

This PR has a migration; here is the generated SQL for src/sentry/migrations/0363_debug_id_artifact_bundle.py ()

--
-- Create model ArtifactBundle
--
CREATE TABLE "sentry_artifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "bundle_id" uuid NULL, "artifact_count" integer NOT NULL CHECK ("artifact_count" >= 0), "date_added" timestamp with time zone NOT NULL, "file_id" bigint NOT NULL);
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id" FOREIGN KEY ("file_id") REFERENCES "sentry_file" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_artifactbundle" VALIDATE CONSTRAINT "sentry_artifactbundle_file_id_1d6752c9_fk_sentry_file_id";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" ON "sentry_artifactbundle" ("organization_id", "bundle_id");
ALTER TABLE "sentry_artifactbundle" ADD CONSTRAINT "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq" UNIQUE USING INDEX "sentry_artifactbundle_organization_id_bundle_id_19f21a4b_uniq";
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_organization_id_b5a13770" ON "sentry_artifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_artifactbundle_file_id_1d6752c9" ON "sentry_artifactbundle" ("file_id");
--
-- Create model ReleaseArtifactBundle
--
CREATE TABLE "sentry_releaseartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "release_name" varchar(250) NOT NULL, "dist_id" bigint NULL, "date_added" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_releaseartifactbundle" VALIDATE CONSTRAINT "sentry_releaseartifa_artifact_bundle_id_e0298a12_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" ON "sentry_releaseartifactbundle" ("organization_id", "release_name", "dist_id", "artifact_bundle_id");
ALTER TABLE "sentry_releaseartifactbundle" ADD CONSTRAINT "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq" UNIQUE USING INDEX "sentry_releaseartifactbu_organization_id_release__ebde7736_uniq";
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_organization_id_10fc7929" ON "sentry_releaseartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_releaseartifactbundle_artifact_bundle_id_e0298a12" ON "sentry_releaseartifactbundle" ("artifact_bundle_id");
--
-- Create model ProjectArtifactBundle
--
CREATE TABLE "sentry_projectartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "project_id" bigint NOT NULL, "date_added" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_projectartifactbundle" VALIDATE CONSTRAINT "sentry_projectartifa_artifact_bundle_id_c3f68015_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" ON "sentry_projectartifactbundle" ("project_id", "artifact_bundle_id");
ALTER TABLE "sentry_projectartifactbundle" ADD CONSTRAINT "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq" UNIQUE USING INDEX "sentry_projectartifactbu_project_id_artifact_bund_b374329e_uniq";
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_organization_id_a4b9d5f9" ON "sentry_projectartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_project_id_7588084b" ON "sentry_projectartifactbundle" ("project_id");
CREATE INDEX CONCURRENTLY "sentry_projectartifactbundle_artifact_bundle_id_c3f68015" ON "sentry_projectartifactbundle" ("artifact_bundle_id");
--
-- Create model DebugIdArtifactBundle
--
CREATE TABLE "sentry_debugidartifactbundle" ("id" bigserial NOT NULL PRIMARY KEY, "organization_id" bigint NOT NULL, "debug_id" uuid NOT NULL, "source_file_type" integer NOT NULL, "date_added" timestamp with time zone NOT NULL, "date_last_accessed" timestamp with time zone NOT NULL, "artifact_bundle_id" bigint NOT NULL);
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar" FOREIGN KEY ("artifact_bundle_id") REFERENCES "sentry_artifactbundle" ("id") DEFERRABLE INITIALLY DEFERRED NOT VALID;
ALTER TABLE "sentry_debugidartifactbundle" VALIDATE CONSTRAINT "sentry_debugidartifa_artifact_bundle_id_91f3140f_fk_sentry_ar";
CREATE UNIQUE INDEX CONCURRENTLY "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" ON "sentry_debugidartifactbundle" ("debug_id", "artifact_bundle_id", "source_file_type");
ALTER TABLE "sentry_debugidartifactbundle" ADD CONSTRAINT "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq" UNIQUE USING INDEX "sentry_debugidartifactbu_debug_id_artifact_bundle_0e34c4ff_uniq";
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_organization_id_4db980bf" ON "sentry_debugidartifactbundle" ("organization_id");
CREATE INDEX CONCURRENTLY "sentry_debugidartifactbundle_artifact_bundle_id_91f3140f" ON "sentry_debugidartifactbundle" ("artifact_bundle_id");