From 0839901e8246115af6e71fa01e8070971a0137ea Mon Sep 17 00:00:00 2001 From: rorepo Date: Thu, 2 Jul 2015 23:03:36 +0100 Subject: [PATCH 01/15] Fake data - subscriptions and payments --- gratipay/utils/fake_data.py | 86 ++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 7a33d62621..2d0931897c 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -211,6 +211,22 @@ def fake_elsewhere(db, participant, platform): , extra_info=None ) +def fake_payment(db, participant, team): + """Create fake payment + """ + direction = ['to-team','to-participant'] + if participant.username == team.owner: + direction = 'to-participant' + else: + direction = random.sample(['to-team','to-participant'],1)[0] + return _fake_thing( db + , "payments" + , timestamp=faker.date_time_this_year() + , participant=participant.username + , team=team.slug + , amount=fake_tip_amount() + , direction=direction + ) def fake_transfer(db, tipper, tippee): return insert_fake_data( db @@ -257,6 +273,26 @@ def prep_db(db): CREATE TRIGGER process_transfer AFTER INSERT ON transfers FOR EACH ROW EXECUTE PROCEDURE process_transfer(); + CREATE OR REPLACE FUNCTION process_payment() RETURNS trigger AS $$ + BEGIN + UPDATE participants + SET balance = balance + NEW.amount + WHERE username = NEW.participant + AND NEW.direction = 'to-participant'; + + UPDATE participants + SET balance = balance - NEW.amount + WHERE username = NEW.participant + AND NEW.direction = 'to-team'; + + RETURN NULL; + + END; + $$ language plpgsql; + + CREATE TRIGGER process_payment AFTER INSERT ON payments + FOR EACH ROW EXECUTE PROCEDURE process_payment(); + CREATE OR REPLACE FUNCTION process_exchange() RETURNS trigger AS $$ BEGIN IF NEW.amount > 0 THEN @@ -331,7 +367,6 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= if npayment_instructions > ntips: break - print("Making Elsewheres") for p in participants: #All participants get between 1 and 3 elsewheres @@ -354,6 +389,22 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= tipper, tippee = random.sample(participants, 2) tips.append(fake_tip(db, tipper, tippee)) + # Payments + payments = [] + paymentcount = 0 + while paymentcount <= num_payments: + for participant in participants: + for team in teams: + paymentcount += 1 + if paymentcount > num_payments: + break + sys.stdout.write("\rMaking Payments (%i/%i)" % (paymentcount, num_payments)) + sys.stdout.flush() + payments.append(fake_payment(db, participant, team)) + if paymentcount > num_payments: + break + print("") + # Transfers transfers = [] for i in xrange(num_transfers): @@ -365,9 +416,13 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Paydays # First determine the boundaries - min and max date - min_date = min(min(x['ctime'] for x in tips), \ + min_date = min(min(x['ctime'] for x in subscriptions), \ + min(x['timestamp'] for x in payments), \ + min(x['ctime'] for x in tips), \ min(x['timestamp'] for x in transfers)) - max_date = max(max(x['ctime'] for x in tips), \ + max_date = max(max(x['ctime'] for x in subscriptions), \ + max(x['timestamp'] for x in payments), \ + max(x['ctime'] for x in tips), \ max(x['timestamp'] for x in transfers)) # iterate through min_date, max_date one week at a time payday_counter = 1 @@ -378,15 +433,25 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= sys.stdout.flush() payday_counter += 1 end_date = date + datetime.timedelta(days=7) - week_tips = filter(lambda x: date < x['ctime'] < end_date, tips) - week_transfers = filter(lambda x: date < x['timestamp'] < end_date, transfers) + week_tips = filter(lambda x: date <= x['ctime'] < end_date, tips) + week_transfers = filter(lambda x: date <= x['timestamp'] < end_date, transfers) + week_subscriptions = filter(lambda x: date <= x['ctime'] < end_date, subscriptions) + week_payments = filter(lambda x: date <= x['timestamp'] < end_date, payments) week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) < end_date, participants) for p in week_participants: transfers_in = filter(lambda x: x['tippee'] == p.username, week_transfers) + payments_in = filter(lambda x: (x['participant'] == p.username) & + (x['direction'] == 'to-participant'), week_payments) transfers_out = filter(lambda x: x['tipper'] == p.username, week_transfers) + payments_out = filter(lambda x: (x['participant'] == p.username) & + (x['direction'] == 'to-team'), week_payments) amount_in = sum([t['amount'] for t in transfers_in]) + amount_in = amount_in + sum([t['amount'] for t in payments_in]) amount_out = sum([t['amount'] for t in transfers_out]) + amount_out = amount_out + sum([t['amount'] for t in payments_out]) amount = amount_out - amount_in + fee = amount * D('0.02') + fee = abs(fee.quantize(D('.01'))) if amount != 0: fee = amount * D('0.02') fee = abs(fee.quantize(D('.01'))) @@ -399,10 +464,21 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= ) actives=set() tippers=set() + #week_tips, week_transfers for xfers in week_tips, week_transfers: actives.update(x['tipper'] for x in xfers) actives.update(x['tippee'] for x in xfers) tippers.update(x['tipper'] for x in xfers) + + # week_subscriptions + actives.update(x['subscriber'] for x in week_subscriptions) + tippers.update(x['subscriber'] for x in week_subscriptions) + + # week_payments + actives.update(x['participant'] for x in week_payments) + tip_payments = filter(lambda x: x['direction'] == 'to-team', week_payments) + tippers.update(x['participant'] for x in tip_payments) + payday = { 'ts_start': date, 'ts_end': end_date, From 03e6a325b2f5daa460af7c7030316f731fbd7106 Mon Sep 17 00:00:00 2001 From: rorepo Date: Fri, 3 Jul 2015 14:42:33 +0100 Subject: [PATCH 02/15] Fake data - payments direction corrected --- gratipay/utils/fake_data.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 2d0931897c..4a286fc0db 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -214,11 +214,10 @@ def fake_elsewhere(db, participant, platform): def fake_payment(db, participant, team): """Create fake payment """ - direction = ['to-team','to-participant'] if participant.username == team.owner: direction = 'to-participant' else: - direction = random.sample(['to-team','to-participant'],1)[0] + direction = 'to-team' return _fake_thing( db , "payments" , timestamp=faker.date_time_this_year() @@ -416,13 +415,13 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Paydays # First determine the boundaries - min and max date - min_date = min(min(x['ctime'] for x in subscriptions), \ - min(x['timestamp'] for x in payments), \ - min(x['ctime'] for x in tips), \ + min_date = min(min(x['ctime'] for x in subscriptions), + min(x['timestamp'] for x in payments), + min(x['ctime'] for x in tips), min(x['timestamp'] for x in transfers)) - max_date = max(max(x['ctime'] for x in subscriptions), \ - max(x['timestamp'] for x in payments), \ - max(x['ctime'] for x in tips), \ + max_date = max(max(x['ctime'] for x in subscriptions), + max(x['timestamp'] for x in payments), + max(x['ctime'] for x in tips), max(x['timestamp'] for x in transfers)) # iterate through min_date, max_date one week at a time payday_counter = 1 From 9d6ad8a837de28c54ae4bb93dbd6d97ff570fd41 Mon Sep 17 00:00:00 2001 From: rorepo Date: Mon, 6 Jul 2015 15:14:10 +0100 Subject: [PATCH 03/15] Fake data - payments with balance checks --- gratipay/utils/fake_data.py | 41 ++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 4a286fc0db..e29b2d6222 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -211,19 +211,15 @@ def fake_elsewhere(db, participant, platform): , extra_info=None ) -def fake_payment(db, participant, team): +def fake_payment(db, participant, team, amount, direction): """Create fake payment """ - if participant.username == team.owner: - direction = 'to-participant' - else: - direction = 'to-team' return _fake_thing( db , "payments" , timestamp=faker.date_time_this_year() - , participant=participant.username - , team=team.slug - , amount=fake_tip_amount() + , participant=participant + , team=team + , amount=amount , direction=direction ) @@ -391,17 +387,24 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Payments payments = [] paymentcount = 0 - while paymentcount <= num_payments: - for participant in participants: - for team in teams: - paymentcount += 1 - if paymentcount > num_payments: - break - sys.stdout.write("\rMaking Payments (%i/%i)" % (paymentcount, num_payments)) - sys.stdout.flush() - payments.append(fake_payment(db, participant, team)) - if paymentcount > num_payments: - break + teamamounts = {} + for team in teams: + teamamounts[team.slug] = 0 + for subscription in subscriptions: + participant = subscription['subscriber'] + team = Team.from_slug(subscription['team']) + amount = subscription['amount'] + if participant != team.owner: + paymentcount += 1 + sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) + sys.stdout.flush() + payments.append(fake_payment(db, participant, team.slug, amount, 'to-team')) + teamamounts[team.slug] = teamamounts[team.slug] + amount + for team in teams: + paymentcount += 1 + sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) + sys.stdout.flush() + payments.append(fake_payment(db, team.owner, team.slug, teamamounts[team.slug], 'to-participant')) print("") # Transfers From a87a53276ae43f71da4222d8ece89af26477945e Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 20:22:59 +0530 Subject: [PATCH 04/15] Increase the number of teams to 25 --- gratipay/utils/fake_data.py | 41 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index e29b2d6222..4718d68446 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -219,7 +219,7 @@ def fake_payment(db, participant, team, amount, direction): , timestamp=faker.date_time_this_year() , participant=participant , team=team - , amount=amount + , amount=amount , direction=direction ) @@ -316,7 +316,12 @@ def clean_db(db): """) +<<<<<<< HEAD def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=5000, num_communities=20): +======= +def populate_db(db, num_participants=100, num_tips=200, num_teams=25, num_transfers=5000, + num_communities=20): +>>>>>>> 7828f7b... Increase the number of teams to 25 """Populate DB with fake data. """ print("Making Participants") @@ -355,12 +360,16 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= for team in teams: #eliminate self-payment if participant.username != team.owner: +<<<<<<< HEAD npayment_instructions += 1 if npayment_instructions > ntips: break fake_payment_instruction(db, participant, team) if npayment_instructions > ntips: break +======= + subscriptions.append(fake_subscription(db, participant, team)) +>>>>>>> 7828f7b... Increase the number of teams to 25 print("Making Elsewheres") for p in participants: @@ -394,7 +403,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= participant = subscription['subscriber'] team = Team.from_slug(subscription['team']) amount = subscription['amount'] - if participant != team.owner: + if participant != team.owner: paymentcount += 1 sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) sys.stdout.flush() @@ -418,13 +427,13 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Paydays # First determine the boundaries - min and max date - min_date = min(min(x['ctime'] for x in subscriptions), - min(x['timestamp'] for x in payments), - min(x['ctime'] for x in tips), + min_date = min(min(x['ctime'] for x in subscriptions), + min(x['timestamp'] for x in payments), + min(x['ctime'] for x in tips), min(x['timestamp'] for x in transfers)) - max_date = max(max(x['ctime'] for x in subscriptions), - max(x['timestamp'] for x in payments), - max(x['ctime'] for x in tips), + max_date = max(max(x['ctime'] for x in subscriptions), + max(x['timestamp'] for x in payments), + max(x['ctime'] for x in tips), max(x['timestamp'] for x in transfers)) # iterate through min_date, max_date one week at a time payday_counter = 1 @@ -442,10 +451,10 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) < end_date, participants) for p in week_participants: transfers_in = filter(lambda x: x['tippee'] == p.username, week_transfers) - payments_in = filter(lambda x: (x['participant'] == p.username) & + payments_in = filter(lambda x: (x['participant'] == p.username) & (x['direction'] == 'to-participant'), week_payments) transfers_out = filter(lambda x: x['tipper'] == p.username, week_transfers) - payments_out = filter(lambda x: (x['participant'] == p.username) & + payments_out = filter(lambda x: (x['participant'] == p.username) & (x['direction'] == 'to-team'), week_payments) amount_in = sum([t['amount'] for t in transfers_in]) amount_in = amount_in + sum([t['amount'] for t in payments_in]) @@ -475,7 +484,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # week_subscriptions actives.update(x['subscriber'] for x in week_subscriptions) tippers.update(x['subscriber'] for x in week_subscriptions) - + # week_payments actives.update(x['participant'] for x in week_payments) tip_payments = filter(lambda x: x['direction'] == 'to-team', week_payments) @@ -484,8 +493,18 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= payday = { 'ts_start': date, 'ts_end': end_date, +<<<<<<< HEAD 'nusers': len(actives), 'volume': sum(x['amount'] for x in week_transfers) +======= + 'ntips': len(week_tips) + len(week_subscriptions), + 'ntransfers': len(week_transfers) + len(week_payments), + 'nparticipants': len(week_participants), + 'ntippers': len(tippers), + 'nactive': len(actives), + 'transfer_volume': sum(x['amount'] for x in week_transfers) + + sum(x['amount'] for x in week_payments) +>>>>>>> 7828f7b... Increase the number of teams to 25 } insert_fake_data(db, "paydays", **payday) date = end_date From d699354e91f12be3ee21dd62bf63aadaed160014 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 20:26:35 +0530 Subject: [PATCH 05/15] Use fake sentences for fields in team model --- gratipay/utils/fake_data.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 4718d68446..a18db07a20 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -316,12 +316,7 @@ def clean_db(db): """) -<<<<<<< HEAD def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=5000, num_communities=20): -======= -def populate_db(db, num_participants=100, num_tips=200, num_teams=25, num_transfers=5000, - num_communities=20): ->>>>>>> 7828f7b... Increase the number of teams to 25 """Populate DB with fake data. """ print("Making Participants") From a6ff387286ccdaae8f962f509575616bd71a7c68 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 20:39:40 +0530 Subject: [PATCH 06/15] Don't fake receiving and nmembers --- gratipay/utils/fake_data.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index a18db07a20..f3a0d5b7e3 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -355,16 +355,12 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= for team in teams: #eliminate self-payment if participant.username != team.owner: -<<<<<<< HEAD npayment_instructions += 1 if npayment_instructions > ntips: break fake_payment_instruction(db, participant, team) if npayment_instructions > ntips: break -======= - subscriptions.append(fake_subscription(db, participant, team)) ->>>>>>> 7828f7b... Increase the number of teams to 25 print("Making Elsewheres") for p in participants: @@ -488,18 +484,8 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= payday = { 'ts_start': date, 'ts_end': end_date, -<<<<<<< HEAD 'nusers': len(actives), 'volume': sum(x['amount'] for x in week_transfers) -======= - 'ntips': len(week_tips) + len(week_subscriptions), - 'ntransfers': len(week_transfers) + len(week_payments), - 'nparticipants': len(week_participants), - 'ntippers': len(tippers), - 'nactive': len(actives), - 'transfer_volume': sum(x['amount'] for x in week_transfers) - + sum(x['amount'] for x in week_payments) ->>>>>>> 7828f7b... Increase the number of teams to 25 } insert_fake_data(db, "paydays", **payday) date = end_date From 5f35597cee75126d5189d79083e691a92825e5df Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 21:34:15 +0530 Subject: [PATCH 07/15] Split week_payments into week_payments_to_teams and week_payments_to_owners --- gratipay/utils/fake_data.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index f3a0d5b7e3..6d79e6c30a 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -394,12 +394,12 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= participant = subscription['subscriber'] team = Team.from_slug(subscription['team']) amount = subscription['amount'] - if participant != team.owner: - paymentcount += 1 - sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) - sys.stdout.flush() - payments.append(fake_payment(db, participant, team.slug, amount, 'to-team')) - teamamounts[team.slug] = teamamounts[team.slug] + amount + assert participant != team.owner + paymentcount += 1 + sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) + sys.stdout.flush() + payments.append(fake_payment(db, participant, team.slug, amount, 'to-team')) + teamamounts[team.slug] = teamamounts[team.slug] + amount for team in teams: paymentcount += 1 sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) @@ -439,14 +439,14 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= week_transfers = filter(lambda x: date <= x['timestamp'] < end_date, transfers) week_subscriptions = filter(lambda x: date <= x['ctime'] < end_date, subscriptions) week_payments = filter(lambda x: date <= x['timestamp'] < end_date, payments) + week_payments_to_teams = filter(lambda x: x['direction'] == 'to-team', week_payments) + week_payments_to_owners = filter(lambda x: x['direction'] == 'to-participant', week_payments) week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) < end_date, participants) for p in week_participants: transfers_in = filter(lambda x: x['tippee'] == p.username, week_transfers) - payments_in = filter(lambda x: (x['participant'] == p.username) & - (x['direction'] == 'to-participant'), week_payments) + payments_in = filter(lambda x: x['participant'] == p.username, week_payments_to_owners) transfers_out = filter(lambda x: x['tipper'] == p.username, week_transfers) - payments_out = filter(lambda x: (x['participant'] == p.username) & - (x['direction'] == 'to-team'), week_payments) + payments_out = filter(lambda x: x['participant'] == p.username, week_payments_to_teams) amount_in = sum([t['amount'] for t in transfers_in]) amount_in = amount_in + sum([t['amount'] for t in payments_in]) amount_out = sum([t['amount'] for t in transfers_out]) @@ -478,8 +478,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # week_payments actives.update(x['participant'] for x in week_payments) - tip_payments = filter(lambda x: x['direction'] == 'to-team', week_payments) - tippers.update(x['participant'] for x in tip_payments) + tippers.update(x['participant'] for x in week_payments_to_owners) payday = { 'ts_start': date, From ed164fdc69ec7159ad0b5d23d39cf10eb06a30f2 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 21:39:57 +0530 Subject: [PATCH 08/15] Use default dict for team_amounts --- gratipay/utils/fake_data.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 6d79e6c30a..fd8997f26e 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -387,9 +387,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Payments payments = [] paymentcount = 0 - teamamounts = {} - for team in teams: - teamamounts[team.slug] = 0 + team_amounts = defaultdict(int) for subscription in subscriptions: participant = subscription['subscriber'] team = Team.from_slug(subscription['team']) @@ -399,12 +397,12 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) sys.stdout.flush() payments.append(fake_payment(db, participant, team.slug, amount, 'to-team')) - teamamounts[team.slug] = teamamounts[team.slug] + amount + team_amounts[team.slug] += amount for team in teams: paymentcount += 1 sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) sys.stdout.flush() - payments.append(fake_payment(db, team.owner, team.slug, teamamounts[team.slug], 'to-participant')) + payments.append(fake_payment(db, team.owner, team.slug, team_amounts[team.slug], 'to-participant')) print("") # Transfers From bc696bc836ab3d2189e6e0c04ebdf52890287a8a Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 21:45:21 +0530 Subject: [PATCH 09/15] Simplify calculations for amount_in and amount_out --- gratipay/utils/fake_data.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index fd8997f26e..62e9e3f990 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -445,10 +445,8 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= payments_in = filter(lambda x: x['participant'] == p.username, week_payments_to_owners) transfers_out = filter(lambda x: x['tipper'] == p.username, week_transfers) payments_out = filter(lambda x: x['participant'] == p.username, week_payments_to_teams) - amount_in = sum([t['amount'] for t in transfers_in]) - amount_in = amount_in + sum([t['amount'] for t in payments_in]) - amount_out = sum([t['amount'] for t in transfers_out]) - amount_out = amount_out + sum([t['amount'] for t in payments_out]) + amount_in = sum([t['amount'] for t in transfers_in + payments_in]) + amount_out = sum([t['amount'] for t in transfers_out + payments_out]) amount = amount_out - amount_in fee = amount * D('0.02') fee = abs(fee.quantize(D('.01'))) From 8383a87b737c24ccca39c9733e8022d83ea0637b Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 22:08:48 +0530 Subject: [PATCH 10/15] Don't create subscriptions to all teams from all members --- gratipay/utils/fake_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 62e9e3f990..7f2c38fa3e 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -281,7 +281,6 @@ def prep_db(db): AND NEW.direction = 'to-team'; RETURN NULL; - END; $$ language plpgsql; @@ -313,7 +312,6 @@ def clean_db(db): db.run(""" DROP FUNCTION IF EXISTS process_transfer() CASCADE; DROP FUNCTION IF EXISTS process_exchange() CASCADE; - """) def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=5000, num_communities=20): From 09fa1c27db2403ce50b72e0682a73b4017e61e2f Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 7 Jul 2015 22:27:25 +0530 Subject: [PATCH 11/15] Simplify the end_date and min_date logic --- gratipay/utils/fake_data.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 7f2c38fa3e..96549557a5 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -414,14 +414,10 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Paydays # First determine the boundaries - min and max date - min_date = min(min(x['ctime'] for x in subscriptions), - min(x['timestamp'] for x in payments), - min(x['ctime'] for x in tips), - min(x['timestamp'] for x in transfers)) - max_date = max(max(x['ctime'] for x in subscriptions), - max(x['timestamp'] for x in payments), - max(x['ctime'] for x in tips), - max(x['timestamp'] for x in transfers)) + min_date = min(min(x['ctime'] for x in subscriptions + tips), + min(x['timestamp'] for x in payments + transfers)) + max_date = max(max(x['ctime'] for x in subscriptions + tips), + max(x['timestamp'] for x in payments + transfers)) # iterate through min_date, max_date one week at a time payday_counter = 1 date = min_date From 00a8f80c4a57a9b591f3942139706db2469c9a3a Mon Sep 17 00:00:00 2001 From: rorepo Date: Mon, 13 Jul 2015 22:55:16 +0100 Subject: [PATCH 12/15] Fixed residual negative balances --- gratipay/utils/fake_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 96549557a5..73aa38b674 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -433,8 +433,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= week_payments = filter(lambda x: date <= x['timestamp'] < end_date, payments) week_payments_to_teams = filter(lambda x: x['direction'] == 'to-team', week_payments) week_payments_to_owners = filter(lambda x: x['direction'] == 'to-participant', week_payments) - week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) < end_date, participants) - for p in week_participants: + for p in participants: transfers_in = filter(lambda x: x['tippee'] == p.username, week_transfers) payments_in = filter(lambda x: x['participant'] == p.username, week_payments_to_owners) transfers_out = filter(lambda x: x['tipper'] == p.username, week_transfers) From 961c72043eff091578619f59511fa4b012f29896 Mon Sep 17 00:00:00 2001 From: Kwesi Aguillera Date: Fri, 16 Sep 2016 12:34:56 -0400 Subject: [PATCH 13/15] Fixed broke fake_data and replaced subscriptions with payment_instructions --- gratipay/utils/fake_data.py | 69 +++++++++++++++++++++++++------------ tests/py/test_fake_data.py | 10 ++---- 2 files changed, 50 insertions(+), 29 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 73aa38b674..56ae0d5573 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -5,6 +5,7 @@ import random import string import sys +from collections import defaultdict from faker import Factory from psycopg2 import IntegrityError @@ -84,6 +85,7 @@ def fake_participant(db, is_admin=False, random_identities=True): ) participant = Participant.from_username(username) + fake_exchange_route(db, participant) if random_identities: if random.randrange(100) < 66: fake_participant_identity(participant) @@ -98,6 +100,21 @@ def fake_participant(db, is_admin=False, random_identities=True): return participant +def fake_exchange_route(db, participant, network=None): + + if not network: + networks = ["balanced-ba", "balanced-cc", "paypal", "bitcoin"] + network = random.sample(networks, 1)[0] + + insert_fake_data( db + , "exchange_routes" + , participant = participant.id + , network = network + , address = participant.email_address + , error = "None" + ) + + def fake_participant_identity(participant, verification=None): """Pick a country and make an identity for the participant there. @@ -214,14 +231,14 @@ def fake_elsewhere(db, participant, platform): def fake_payment(db, participant, team, amount, direction): """Create fake payment """ - return _fake_thing( db - , "payments" - , timestamp=faker.date_time_this_year() - , participant=participant - , team=team - , amount=amount - , direction=direction - ) + return insert_fake_data( db + , "payments" + , timestamp=faker.date_time_this_year() + , participant=participant + , team=team + , amount=amount + , direction=direction + ) def fake_transfer(db, tipper, tippee): return insert_fake_data( db @@ -242,13 +259,19 @@ def fake_exchange(db, participant, amount, fee, timestamp): , amount=amount , fee=fee , status='succeeded' + , route=get_exchange_route(db, participant.id) ) +def get_exchange_route(db, participant): + return db.one("SELECT id FROM exchange_routes WHERE participant={}" + .format(participant), participant) + def random_country_id(db): return db.one("SELECT id FROM countries ORDER BY random() LIMIT 1") + def prep_db(db): db.run(""" CREATE OR REPLACE FUNCTION process_transfer() RETURNS trigger AS $$ @@ -312,7 +335,8 @@ def clean_db(db): db.run(""" DROP FUNCTION IF EXISTS process_transfer() CASCADE; DROP FUNCTION IF EXISTS process_exchange() CASCADE; - + DROP FUNCTION IF EXISTS process_payment() CASCADE; + """) def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=5000, num_communities=20): """Populate DB with fake data. @@ -349,6 +373,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= print("Making Payment Instructions") npayment_instructions = 0 + payment_instructions = [] for participant in participants: for team in teams: #eliminate self-payment @@ -356,7 +381,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= npayment_instructions += 1 if npayment_instructions > ntips: break - fake_payment_instruction(db, participant, team) + payment_instructions.append(fake_payment_instruction(db, participant, team)) if npayment_instructions > ntips: break @@ -386,15 +411,15 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= payments = [] paymentcount = 0 team_amounts = defaultdict(int) - for subscription in subscriptions: - participant = subscription['subscriber'] - team = Team.from_slug(subscription['team']) - amount = subscription['amount'] - assert participant != team.owner + for payment_instruction in payment_instructions: + participant = Participant.from_id(payment_instruction['participant_id']) + team = Team.from_id(payment_instruction['team_id']) + amount = payment_instruction['amount'] + assert participant.username != team.owner paymentcount += 1 sys.stdout.write("\rMaking Payments (%i)" % (paymentcount)) sys.stdout.flush() - payments.append(fake_payment(db, participant, team.slug, amount, 'to-team')) + payments.append(fake_payment(db, participant.username, team.slug, amount, 'to-team')) team_amounts[team.slug] += amount for team in teams: paymentcount += 1 @@ -414,9 +439,9 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= # Paydays # First determine the boundaries - min and max date - min_date = min(min(x['ctime'] for x in subscriptions + tips), + min_date = min(min(x['ctime'] for x in payment_instructions + tips), min(x['timestamp'] for x in payments + transfers)) - max_date = max(max(x['ctime'] for x in subscriptions + tips), + max_date = max(max(x['ctime'] for x in payment_instructions + tips), max(x['timestamp'] for x in payments + transfers)) # iterate through min_date, max_date one week at a time payday_counter = 1 @@ -429,7 +454,7 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= end_date = date + datetime.timedelta(days=7) week_tips = filter(lambda x: date <= x['ctime'] < end_date, tips) week_transfers = filter(lambda x: date <= x['timestamp'] < end_date, transfers) - week_subscriptions = filter(lambda x: date <= x['ctime'] < end_date, subscriptions) + week_payment_instructions = filter(lambda x: date <= x['ctime'] < end_date, payment_instructions) week_payments = filter(lambda x: date <= x['timestamp'] < end_date, payments) week_payments_to_teams = filter(lambda x: x['direction'] == 'to-team', week_payments) week_payments_to_owners = filter(lambda x: x['direction'] == 'to-participant', week_payments) @@ -461,9 +486,9 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= actives.update(x['tippee'] for x in xfers) tippers.update(x['tipper'] for x in xfers) - # week_subscriptions - actives.update(x['subscriber'] for x in week_subscriptions) - tippers.update(x['subscriber'] for x in week_subscriptions) + # week_payment_instructions + actives.update(x['participant_id'] for x in week_payment_instructions) + tippers.update(x['participant_id'] for x in week_payment_instructions) # week_payments actives.update(x['participant'] for x in week_payments) diff --git a/tests/py/test_fake_data.py b/tests/py/test_fake_data.py index c39d2845c8..395eb54290 100644 --- a/tests/py/test_fake_data.py +++ b/tests/py/test_fake_data.py @@ -8,11 +8,10 @@ class TestFakeData(Harness): """ Ensure the fake_data script doesn't throw any exceptions """ - def test_fake_data(self): num_participants = 6 - num_tips = 5 - num_teams = 1 + num_tips = 25 + num_teams = 5 num_transfers = 5 fake_data.main(self.db, num_participants, num_tips, num_teams, num_transfers) tips = self.db.all("SELECT * FROM tips") @@ -24,10 +23,7 @@ def test_fake_data(self): assert len(participants) == num_participants assert len(transfers) == num_transfers assert len(teams) == num_teams + 1 # +1 for the fake Gratipay team. - if num_tips <= num_participants - num_teams: - assert len(payment_instructions) == num_tips - else: - assert len(payment_instructions) == (num_participants - num_teams) + assert len(payment_instructions) == num_tips def test_fake_participant_identity(self): From d517f1b065db3413704996240c4a53953f6b15cd Mon Sep 17 00:00:00 2001 From: Kwesi Aguillera Date: Fri, 16 Sep 2016 14:16:55 -0400 Subject: [PATCH 14/15] Removed Community from fake data --- gratipay/utils/fake_data.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/gratipay/utils/fake_data.py b/gratipay/utils/fake_data.py index 56ae0d5573..be3706937c 100644 --- a/gratipay/utils/fake_data.py +++ b/gratipay/utils/fake_data.py @@ -14,7 +14,6 @@ from gratipay.elsewhere import PLATFORMS from gratipay.models.participant import Participant from gratipay.models.team import slugize, Team -from gratipay.models import community from gratipay.models import check_db from gratipay.exceptions import InvalidTeamName @@ -178,20 +177,6 @@ def fake_payment_instruction(db, participant, team): ) -def fake_community(db, creator): - """Create a fake community - """ - name = faker.city() - if not community.name_pattern.match(name): - return fake_community(db, creator) - - slug = community.slugize(name) - - creator.insert_into_communities(True, name, slug) - - return community.Community.from_slug(slug) - - def fake_tip_amount(): amount = ((D(random.random()) * (MAX_TIP - MIN_TIP)) + MIN_TIP) @@ -338,7 +323,7 @@ def clean_db(db): DROP FUNCTION IF EXISTS process_payment() CASCADE; """) -def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=5000, num_communities=20): +def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers=5000): """Populate DB with fake data. """ print("Making Participants") @@ -392,14 +377,6 @@ def populate_db(db, num_participants=100, ntips=200, num_teams=5, num_transfers= for platform_name in random.sample(PLATFORMS, num_elsewheres): fake_elsewhere(db, p, platform_name) - print("Making Communities") - for i in xrange(num_communities): - creator = random.sample(participants, 1) - community = fake_community(db, creator[0]) - - members = random.sample(participants, random.randint(1, 3)) - for p in members: - p.insert_into_communities(True, community.name, community.slug) print("Making Tips") tips = [] From b57112c9b4f197cc59b5c6c9a522890ee8ff12ab Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 25 Aug 2016 08:54:16 -0400 Subject: [PATCH 15/15] Only start a browser if we have to --- gratipay/testing/browser.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/gratipay/testing/browser.py b/gratipay/testing/browser.py index 139ceb766f..967be288c0 100644 --- a/gratipay/testing/browser.py +++ b/gratipay/testing/browser.py @@ -13,14 +13,6 @@ from .harness import Harness -def get_browser(driver_name): - # Override to clean up after ourselves properly. - DriverClass = _DRIVERS[driver_name] - driver = DriverClass() - atexit.register(driver.quit) - return driver - - class BrowserHarness(Harness): """This is a harness for through-the-web (TTW) testing. It passes everything through to an underlying `Splinter`_ browser, with the following @@ -30,10 +22,20 @@ class BrowserHarness(Harness): """ - _browser = get_browser(os.environ['WEBDRIVER_BROWSER']) + _browser = None use_VCR = False # without this we get fixture spam from communication with PhantomJS base_url = os.environ['WEBDRIVER_BASE_URL'] + @classmethod + def setUpClass(cls): + super(BrowserHarness, cls).setUpClass() + + # starting a browser is expensive, so we do so lazily, and once + if cls._browser is None: + DriverClass = _DRIVERS[os.environ['WEBDRIVER_BROWSER']] + cls._browser = DriverClass() + atexit.register(cls._browser.quit) + def setUp(self): Harness.setUp(self) self.cookies.delete()