Skip to content

Commit

Permalink
feat(InvoiceError) - Add InvoiceError (#2763)
Browse files Browse the repository at this point in the history
## Description

When invoices fails to generate it's hard to understand what's going
wrong.
We're now automatically retrying invoices that are stuck in generating
state. We will also create InvoiceError objects for invoices that fail
to generate and finalize.
  • Loading branch information
nudded authored Oct 31, 2024
1 parent 8220622 commit be2422e
Show file tree
Hide file tree
Showing 11 changed files with 238 additions and 12 deletions.
8 changes: 4 additions & 4 deletions app/jobs/bill_subscription_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ def perform(subscriptions, timestamp, invoicing_reason:, invoice: nil, skip_char
return if tax_error?(result)

# If the invoice was passed as an argument, it means the job was already retried (see end of function)
result.raise_if_error! if invoice

# If the invoice is in a retryable state, we'll re-enqueue the job manually, otherwise the job fails
result.raise_if_error! unless result.invoice&.generating?
if invoice || !result.invoice&.generating?
InvoiceError.create_for(invoice: result.invoice, error: result.error)
return result.raise_if_error!
end

# On billing day, we'll retry the job further in the future because the system is typically under heavy load
is_billing_date = invoicing_reason.to_sym == :subscription_periodic
Expand Down
26 changes: 26 additions & 0 deletions app/jobs/clock/retry_generating_subscription_invoices_job.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# frozen_string_literal: true

module Clock
class RetryGeneratingSubscriptionInvoicesJob < ApplicationJob
include SentryCronConcern

queue_as 'clock'

THRESHOLD = -> { 1.day.ago }

def perform
Invoice.subscription.generating.where.not(id: InvoiceError.select(:id)).where('created_at < ?', THRESHOLD.call).find_each do |invoice|
next unless invoice.invoice_subscriptions.any?
invoicing_reasons = invoice.invoice_subscriptions.pluck(:invoicing_reason).uniq
invoicing_reason = (invoicing_reasons.size == 1) ? invoicing_reasons.first : :upgrading
BillSubscriptionJob.perform_later(
subscriptions: invoice.subscriptions.to_a,
timestamp: invoice.invoice_subscriptions.first.timestamp,
invoicing_reason:,
invoice:,
skip_charges: invoice.skip_charges
)
end
end
end
end
11 changes: 5 additions & 6 deletions app/models/clickhouse/events_raw.rb
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,12 @@ def organization
#
# Table name: events_raw
#
# code :string not null, primary key
# ingested_at :datetime not null
# code :string not null
# precise_total_amount_cents :decimal(40, 15)
# properties :string not null
# timestamp :datetime not null, primary key
# timestamp :datetime not null
# external_customer_id :string not null
# external_subscription_id :string not null, primary key
# organization_id :string not null, primary key
# transaction_id :string not null, primary key
# external_subscription_id :string not null
# organization_id :string not null
# transaction_id :string not null
#
27 changes: 27 additions & 0 deletions app/models/invoice_error.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# frozen_string_literal: true

class InvoiceError < ApplicationRecord
# NOTE! Invoice errors will have the same id as the invoice they belong to.
def self.create_for(invoice:, error:)
return unless invoice

create(id: invoice.id,
backtrace: error.backtrace,
error: error.inspect.to_json,
invoice: invoice.to_json(except: :file),
subscriptions: invoice.subscriptions.to_json)
end
end

# == Schema Information
#
# Table name: invoice_errors
#
# id :uuid not null, primary key
# backtrace :text
# error :json
# invoice :json
# subscriptions :json
# created_at :datetime not null
# updated_at :datetime not null
#
5 changes: 5 additions & 0 deletions clock.rb
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ module Clockwork
.set(sentry: {"slug" => 'lago_bill_customers', "cron" => '10 */1 * * *'})
.perform_later
end
every(1.hour, 'schedule:retry_generating_subscription_invoices', at: '*:30') do
Clock::RetryGeneratingSubscriptionInvoicesJob
.set(sentry: {"slug" => 'lago_retry_invoices', "cron" => '30 */1 * * *'})
.perform_later
end

every(1.hour, 'schedule:finalize_invoices', at: '*:20') do
Clock::FinalizeInvoicesJob
Expand Down
14 changes: 14 additions & 0 deletions db/migrate/20241031102231_create_invoice_errors.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# frozen_string_literal: true

class CreateInvoiceErrors < ActiveRecord::Migration[7.1]
def change
create_table :invoice_errors, id: :uuid do |t|
t.text :backtrace
t.json :invoice
t.json :subscriptions
t.json :error

t.timestamps
end
end
end
11 changes: 10 additions & 1 deletion db/schema.rb

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 21 additions & 0 deletions spec/clockwork_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,27 @@
end
end

describe 'schedule:retry_generating_subscription_invoices' do
let(:job) { 'schedule:retry_generating_subscription_invoices' }
let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') }
let(:end_time) { Time.zone.parse('1 Apr 2022 01:01:00') }

it 'enqueues a Clock::RetryGeneratingSubscriptionInvoiceJob' do
Clockwork::Test.run(
file: clock_file,
start_time:,
end_time:,
tick_speed: 1.second
)

expect(Clockwork::Test).to be_ran_job(job)
expect(Clockwork::Test.times_run(job)).to eq(1)

Clockwork::Test.block_for(job).call
expect(Clock::RetryGeneratingSubscriptionInvoicesJob).to have_been_enqueued
end
end

describe 'schedule:compute_daily_usage' do
let(:job) { 'schedule:compute_daily_usage' }
let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') }
Expand Down
22 changes: 21 additions & 1 deletion spec/jobs/bill_subscription_job_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@

context 'when result is a failure' do
let(:result) do
BaseService::Result.new.single_validation_failure!(error_code: 'error')
result = BaseService::Result.new
result.invoice = invoice
result.single_validation_failure!(error_code: 'error')
end

it 'raises an error' do
Expand Down Expand Up @@ -57,6 +59,15 @@

expect(Invoices::SubscriptionService).to have_received(:call)
end

it 'creates an InvoiceError' do
expect do
described_class.perform_now(subscriptions, timestamp, invoicing_reason:, invoice:)
end.to raise_error(BaseService::FailedResult)

expect(InvoiceError.all.size).to eq(1)
expect(InvoiceError.first.id).to eq(invoice.id)
end
end

context 'when a generating invoice is attached to the result' do
Expand Down Expand Up @@ -86,6 +97,15 @@

expect(Invoices::SubscriptionService).to have_received(:call)
end

it 'creates an InvoiceError' do
expect do
described_class.perform_now(subscriptions, timestamp, invoicing_reason:)
end.to raise_error(BaseService::FailedResult)

expect(InvoiceError.all.size).to eq(1)
expect(InvoiceError.first.id).to eq(result_invoice.id)
end
end
end
end
63 changes: 63 additions & 0 deletions spec/jobs/clock/retry_generating_subscription_invoices_job_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# frozen_string_literal: true

require 'rails_helper'

describe Clock::RetryGeneratingSubscriptionInvoicesJob, job: true do
subject { described_class }

describe '.perform' do
let(:old_generating_invoice) { create(:invoice, :generating, created_at: 5.days.ago) }

before do
old_generating_invoice
end

it "does not enqueue a BillSubscriptionJob for this invoice (missing subscriptions)" do
expect do
described_class.perform_now
end.not_to have_enqueued_job(BillSubscriptionJob)
end

context "with an actual invoice that should be retried" do
let(:old_generating_invoice) { create(:invoice, :subscription, created_at: 5.days.ago) }

before do
old_generating_invoice.update(status: :generating)
end

it "does enqueue a BillSubscriptionJob for this invoice " do
expect do
described_class.perform_now
end.to have_enqueued_job(BillSubscriptionJob)
end

context "with an existing invoice error" do
let(:invoice_error) { InvoiceError.create(id: old_generating_invoice.id) }

before do
invoice_error
end

it "does not enqueue a BillSubscriptionJob for this invoice" do
expect do
described_class.perform_now
end.not_to have_enqueued_job(BillSubscriptionJob)
end
end
end

context "with an addon" do
let(:old_generating_invoice) { create(:invoice, :add_on, created_at: 5.days.ago) }

before do
old_generating_invoice.update(status: :generating)
end

it "does not enqueue a BillSubscriptionJob for this invoice" do
expect do
described_class.perform_now
end.not_to have_enqueued_job(BillSubscriptionJob)
end
end
end
end
42 changes: 42 additions & 0 deletions spec/models/invoice_error_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# frozen_string_literal: true

require 'rails_helper'

RSpec.describe InvoiceError, type: :model do
let(:invoice) { create(:invoice, :generating) }
let(:result) { BaseService::Result.new }
let(:error) { BaseService::ValidationFailure.new(result, messages: messages) }
let(:messages) { ["message1", "message2"] }

let(:error_with_backtrace) do
error = OpenStruct.new
error.backtrace = "backtrace"
error
end

describe ".create_for" do
it "does nothing if the invoice is nil" do
expect(described_class.create_for(invoice: nil, error:)).to eq(nil)
end

it "creates an invoice error with the same id as the invoice" do
invoice_error = described_class.create_for(invoice:, error:)
expect(invoice_error.id).to eq(invoice.id)
end

it "stores the error in the error field" do
invoice_error = described_class.create_for(invoice:, error:)
expect(invoice_error.error).to eq(error.inspect.to_json)
end

it "stores the backtrace in the backtrace field" do
invoice_error = described_class.create_for(invoice:, error: error_with_backtrace)
expect(invoice_error.backtrace).to eq("backtrace")
end

it "stores the subscriptions in the subscriptions field" do
invoice_error = described_class.create_for(invoice:, error:)
expect(invoice_error.subscriptions).to eq("[]")
end
end
end

0 comments on commit be2422e

Please sign in to comment.