Skip to content

Commit

Permalink
Fb/tenant upgrade resilience (#85)
Browse files Browse the repository at this point in the history 
* registry update resilience

* better filters for tenant update

* expose filters to user

* fix stale filter

* order consistency

* add subscription filters to help

* slightly better wording

* spread new syntax

* fix tests

* rely on proper defaulting in tests

* only filter also for lists

* fix tests

* some tests for new filters

* remove unneeded import

* remove unneeded snapshots

* more precise documentation

* more compact filter code
  • Loading branch information
@rlindner81
rlindner81 authored Jul 22, 2024
1 parent 0992ec3 commit 12bf069
Show file tree
Hide file tree
Showing 8 changed files with 268 additions and 69 deletions.
2 changes: 2 additions & 0 deletions docs/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ commands:
... [TENANT] filter list for tenant id or subdomain
... --time list includes timestamps
... --skip-unchanged skip update for unchanged dependencies
... --only-stale only update subscriptions that have not changed today
... --only-failed only update subscriptions with UPDATE_FAILED state
=== cap multitenancy (cds) ===
~ cdsl --cds-list [TENANT] list all cds-mtx tenant names
Expand Down
2 changes: 2 additions & 0 deletions docs/tenant-registry/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ Commands for this area are:
... [TENANT] filter list for tenant id or subdomain
... --time list includes timestamps
... --skip-unchanged skip update for unchanged dependencies
... --only-stale only update subscriptions that have not changed today
... --only-failed only update subscriptions with UPDATE_FAILED state
~ are read-only commands
* are potentially _dangerous_ commands
Expand Down
10 changes: 8 additions & 2 deletions src/cliOptions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ const FLAG_ARG = Object.freeze({
USER_INFO: "--userinfo",
AUTO_UNDEPLOY: "--auto-undeploy",
SKIP_UNCHANGED: "--skip-unchanged",
ONLY_STALE: "--only-stale",
ONLY_FAILED: "--only-failed",
});

const FORCE_FLAG = "--force";
Expand Down Expand Up @@ -83,6 +85,8 @@ commands:
... [TENANT] filter list for tenant id or subdomain
... --time list includes timestamps
... --skip-unchanged skip update for unchanged dependencies
... --only-stale only update subscriptions that have not changed today
... --only-failed only update subscriptions with UPDATE_FAILED state
=== cap multitenancy (cds) ===
~ cdsl --cds-list [TENANT] list all cds-mtx tenant names
Expand Down Expand Up @@ -209,13 +213,14 @@ const APP_CLI_OPTIONS = Object.freeze({
REGISTRY_LIST: {
commandVariants: ["regl", "--registry-list"],
optionalPassArgs: [PASS_ARG.TENANT],
optionalFlagArgs: [FLAG_ARG.TIMESTAMPS],
optionalFlagArgs: [FLAG_ARG.TIMESTAMPS, FLAG_ARG.ONLY_STALE, FLAG_ARG.ONLY_FAILED],
callback: reg.registryListSubscriptions,
readonly: true,
},
REGISTRY_LONG_LIST: {
commandVariants: ["regll", "--registry-long-list"],
optionalPassArgs: [PASS_ARG.TENANT],
optionalFlagArgs: [FLAG_ARG.ONLY_STALE, FLAG_ARG.ONLY_FAILED],
callback: reg.registryLongListSubscriptions,
readonly: true,
},
Expand All @@ -238,12 +243,13 @@ const APP_CLI_OPTIONS = Object.freeze({
},
REGISTRY_UPDATE_ALL_DEPENDENCIES: {
commandVariants: ["--registry-update-all"],
optionalFlagArgs: [FLAG_ARG.SKIP_UNCHANGED],
optionalFlagArgs: [FLAG_ARG.SKIP_UNCHANGED, FLAG_ARG.ONLY_STALE, FLAG_ARG.ONLY_FAILED],
callback: reg.registryUpdateAllDependencies,
},
REGISTRY_UPDATE_APP_URL: {
commandVariants: ["--registry-update-url"],
optionalPassArgs: [PASS_ARG.TENANT_ID],
optionalFlagArgs: [FLAG_ARG.ONLY_STALE, FLAG_ARG.ONLY_FAILED],
callback: reg.registryUpdateApplicationURL,
},
REGISTRY_OFFBOARD_SUBSCRIPTION: {
Expand Down
5 changes: 3 additions & 2 deletions src/shared/static.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ const nextFreePort = async (port) => {
}
};

const _dateDiffInDays = (from, to) => {
const dateDiffInDays = (from, to) => {
const fromDate = Date.UTC(from.getFullYear(), from.getMonth(), from.getDate());
const toDate = Date.UTC(to.getFullYear(), to.getMonth(), to.getDate());
return Math.floor((toDate - fromDate) / 1000 / 60 / 60 / 24);
Expand All @@ -220,7 +220,7 @@ const formatTimestampWithRelativeDays = (input, nowDate = new Date()) => {
return "";
}
const inputDate = new Date(input);
const daysAgo = _dateDiffInDays(inputDate, nowDate);
const daysAgo = dateDiffInDays(inputDate, nowDate);
const outputAbsolute = inputDate.toISOString().replace(/\.[0-9]{3}/, "");
return `${outputAbsolute} (${daysAgo} ${daysAgo === 1 ? "day" : "days"} ago)`;
};
Expand Down Expand Up @@ -396,6 +396,7 @@ module.exports = {
compareFor,
partition,
spawnAsync,
dateDiffInDays,
formatTimestampWithRelativeDays,
formatTimestampsWithRelativeDays,
resolveTenantArg,
Expand Down
76 changes: 53 additions & 23 deletions src/submodules/tenantRegistry.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ const {
isDashedWord,
sleep,
tableList,
dateDiffInDays,
formatTimestampsWithRelativeDays,
resolveTenantArg,
limiter,
Expand All @@ -24,20 +25,24 @@ const { request } = require("../shared/request");
const REGISTRY_PAGE_SIZE = 200;
const REGISTRY_JOB_POLL_FREQUENCY_FALLBACK = 15000;
const REGISTRY_REQUEST_CONCURRENCY_FALLBACK = 10;
const TENANT_UPDATABLE_STATES = ["SUBSCRIBED", "UPDATE_FAILED"];
const JOB_STATE = Object.freeze({
STARTED: "STARTED",
SUCCEEDED: "SUCCEEDED",
FAILED: "FAILED",
});
const SUBSCRIPTION_STATE = Object.freeze({
SUBSCRIBED: "SUBSCRIBED",
UPDATE_FAILED: "UPDATE_FAILED",
});
const UPDATABLE_STATES = [SUBSCRIPTION_STATE.SUBSCRIBED, SUBSCRIPTION_STATE.UPDATE_FAILED];

const regRequestConcurrency = parseIntWithFallback(
process.env[ENV.REG_CONCURRENCY],
REGISTRY_REQUEST_CONCURRENCY_FALLBACK
);
const regPollFrequency = parseIntWithFallback(process.env[ENV.REG_FREQUENCY], REGISTRY_JOB_POLL_FREQUENCY_FALLBACK);

const _registrySubscriptionsPaged = async (context, tenant) => {
const _registrySubscriptionsPaged = async (context, { tenant, onlyFailed, onlyStale, onlyUpdatable } = {}) => {
const { subdomain: filterSubdomain, tenantId: filterTenantId } = resolveTenantArg(tenant);
filterSubdomain && assert(isDashedWord(filterSubdomain), `argument "${filterSubdomain}" is not a valid subdomain`);

Expand Down Expand Up @@ -73,15 +78,23 @@ const _registrySubscriptionsPaged = async (context, tenant) => {
}
}

if (filterSubdomain) {
subscriptions = subscriptions.filter(({ subdomain }) => subdomain === filterSubdomain);
}
subscriptions = subscriptions.filter(
({ state, subdomain, changedOn }) =>
(!filterSubdomain || subdomain === filterSubdomain) &&
(!onlyFailed || state === SUBSCRIPTION_STATE.UPDATE_FAILED) &&
(!onlyUpdatable || UPDATABLE_STATES.includes(state)) &&
(!onlyStale || dateDiffInDays(new Date(changedOn), new Date()) > 0)
);

return { subscriptions };
};

const registryListSubscriptions = async (context, [tenant], [doTimestamps]) => {
const { subscriptions } = await _registrySubscriptionsPaged(context, tenant);
const registryListSubscriptions = async (context, [tenant], [doTimestamps, doOnlyStale, doOnlyFailed]) => {
const { subscriptions } = await _registrySubscriptionsPaged(context, {
tenant,
onlyStale: doOnlyStale,
onlyFailed: doOnlyFailed,
});
const headerRow = ["consumerTenantId", "globalAccountId", "subdomain", "plan", "state", "url"];
doTimestamps && headerRow.push("created_on", "updated_on");
const nowDate = new Date();
Expand All @@ -103,8 +116,8 @@ const registryListSubscriptions = async (context, [tenant], [doTimestamps]) => {
return tableList(table, { withRowNumber: !tenant });
};

const registryLongListSubscriptions = async (context, [tenant]) => {
const data = await _registrySubscriptionsPaged(context, tenant);
const registryLongListSubscriptions = async (context, [tenant], [doOnlyStale, doOnlyFailed]) => {
const data = await _registrySubscriptionsPaged(context, { tenant, onlyStale: doOnlyStale, onlyFailed: doOnlyFailed });
return JSON.stringify(data, null, 2);
};

Expand Down Expand Up @@ -180,13 +193,18 @@ const _registryCallForTenant = async (
? `/saas-manager/v1/${plan}/subscriptions/${subscriptionId}`
: `/saas-manager/v1/${plan}/tenants/${tenantId}/subscriptions`;
const token = await context.getCachedUaaTokenFromCredentials(credentials);
const response = await request({
method,
url: saas_registry_url,
pathname,
...(Object.keys(query).length !== 0 && { query }),
auth: { token },
});
let response;
try {
response = await request({
method,
url: saas_registry_url,
pathname,
...(Object.keys(query).length !== 0 && { query }),
auth: { token },
});
} catch (err) {
return { tenantId, state: JOB_STATE.FAILED, message: err.message };
}

if (!doJobPoll) {
// NOTE: with checkStatus being true by default, the above request only returns for successful changes
Expand All @@ -206,15 +224,21 @@ const _registryCall = async (context, method, tenantId, options) => {
let results;
if (tenantId) {
assert(isUUID(tenantId), "TENANT_ID is not a uuid", tenantId);
const { subscriptions } = await _registrySubscriptionsPaged(context, tenantId);
const { subscriptions } = await _registrySubscriptionsPaged(context, {
tenant: tenantId,
});
assert(subscriptions.length >= 1, "could not find tenant %s", tenantId);
results = [await _registryCallForTenant(context, subscriptions[0], method, options)];
} else {
const { subscriptions } = await _registrySubscriptionsPaged(context);
const updatableSubscriptions = subscriptions.filter(({ state }) => TENANT_UPDATABLE_STATES.includes(state));
const { onlyStaleSubscriptions, onlyFailedSubscriptions } = options ?? {};
const { subscriptions } = await _registrySubscriptionsPaged(context, {
onlyFailed: onlyFailedSubscriptions,
onlyStale: onlyStaleSubscriptions,
onlyUpdatable: true,
});
results = await limiter(
regRequestConcurrency,
updatableSubscriptions,
subscriptions,
async (subscription) => await _registryCallForTenant(context, subscription, method, options)
);
}
Expand All @@ -230,14 +254,20 @@ const _registryCall = async (context, method, tenantId, options) => {
const registryUpdateDependencies = async (context, [tenantId], [doSkipUnchanged]) =>
await _registryCall(context, "PATCH", tenantId, { skipUnchangedDependencies: doSkipUnchanged });

const registryUpdateAllDependencies = async (context, _, [doSkipUnchanged]) =>
await _registryCall(context, "PATCH", undefined, { skipUnchangedDependencies: doSkipUnchanged });
const registryUpdateAllDependencies = async (context, _, [doSkipUnchanged, doOnlyStale, doOnlyFailed]) =>
await _registryCall(context, "PATCH", undefined, {
skipUnchangedDependencies: doSkipUnchanged,
onlyStaleSubscriptions: doOnlyStale,
onlyFailedSubscriptions: doOnlyFailed,
});

const registryUpdateApplicationURL = async (context, [tenantId]) =>
const registryUpdateApplicationURL = async (context, [tenantId], [doOnlyStale, doOnlyFailed]) =>
await _registryCall(context, "PATCH", tenantId, {
updateApplicationURL: true,
skipUpdatingDependencies: true,
doJobPoll: false,
onlyStaleSubscriptions: doOnlyStale,
onlyFailedSubscriptions: doOnlyFailed,
});
const registryOffboardSubscription = async (context, [tenantId]) => await _registryCall(context, "DELETE", tenantId);

Expand Down
Loading

0 comments on commit 12bf069

Please sign in to comment.