Schedule configurable completed pricing-job cleanup in the worker, broaden repository cleanup across sellers, add unit and integration coverage, and update pricing operational documentation.
This commit is contained in:
@@ -9,6 +9,8 @@ PRICING_FIXTURE_PROVIDER_ENABLED=false
|
|||||||
PRICING_HTTP_PROVIDER_ID=http-provider
|
PRICING_HTTP_PROVIDER_ID=http-provider
|
||||||
PRICING_HTTP_PROVIDER_ENDPOINT=
|
PRICING_HTTP_PROVIDER_ENDPOINT=
|
||||||
PRICING_HTTP_PROVIDER_TOKEN=
|
PRICING_HTTP_PROVIDER_TOKEN=
|
||||||
|
PRICING_JOB_RETENTION_DAYS=30
|
||||||
|
PRICING_JOB_CLEANUP_INTERVAL_MS=3600000
|
||||||
PASSWORD_RESET_URL=http://localhost:5173/reset-password
|
PASSWORD_RESET_URL=http://localhost:5173/reset-password
|
||||||
SMTP_FROM=no-reply@example.test
|
SMTP_FROM=no-reply@example.test
|
||||||
SMTP_URL=smtp://localhost:1025
|
SMTP_URL=smtp://localhost:1025
|
||||||
|
|||||||
@@ -164,12 +164,11 @@ export function createPricingRefreshJobRepository(db) {
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
async purgeCompletedBefore(sellerId, cutoff) {
|
async purgeCompletedBefore(cutoff) {
|
||||||
const deleted = await db
|
const deleted = await db
|
||||||
.delete(pricingRefreshJobs)
|
.delete(pricingRefreshJobs)
|
||||||
.where(
|
.where(
|
||||||
and(
|
and(
|
||||||
eq(pricingRefreshJobs.sellerId, sellerId),
|
|
||||||
eq(pricingRefreshJobs.status, "COMPLETED"),
|
eq(pricingRefreshJobs.status, "COMPLETED"),
|
||||||
lte(pricingRefreshJobs.updatedAt, cutoff),
|
lte(pricingRefreshJobs.updatedAt, cutoff),
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -41,11 +41,12 @@ const pricingRefreshJobRepository = createPricingRefreshJobRepository(
|
|||||||
const providerMappingRepository = createProviderMappingRepository(database.db);
|
const providerMappingRepository = createProviderMappingRepository(database.db);
|
||||||
const saleRepository = createSaleRepository(database.db);
|
const saleRepository = createSaleRepository(database.db);
|
||||||
|
|
||||||
async function createFixture() {
|
async function createFixture(suffix = "") {
|
||||||
|
const fixtureName = suffix ? `transactions-${suffix}` : "transactions";
|
||||||
const [user] = await database.db
|
const [user] = await database.db
|
||||||
.insert(users)
|
.insert(users)
|
||||||
.values({
|
.values({
|
||||||
email: "transactions@example.test",
|
email: `${fixtureName}@example.test`,
|
||||||
passwordHash: "not-a-real-password",
|
passwordHash: "not-a-real-password",
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
@@ -53,14 +54,14 @@ async function createFixture() {
|
|||||||
.insert(sellers)
|
.insert(sellers)
|
||||||
.values({
|
.values({
|
||||||
userId: user.id,
|
userId: user.id,
|
||||||
displayName: "Transactions Owner",
|
displayName: `${fixtureName} Owner`,
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
const [category] = await database.db
|
const [category] = await database.db
|
||||||
.insert(categories)
|
.insert(categories)
|
||||||
.values({
|
.values({
|
||||||
name: "Transactions Cards",
|
name: `${fixtureName} Cards`,
|
||||||
slug: "transactions-cards",
|
slug: `${fixtureName}-cards`,
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
const [profile] = await database.db
|
const [profile] = await database.db
|
||||||
@@ -84,8 +85,8 @@ async function createFixture() {
|
|||||||
.values({
|
.values({
|
||||||
sellerId: seller.id,
|
sellerId: seller.id,
|
||||||
categoryId: category.id,
|
categoryId: category.id,
|
||||||
name: "Integration Card",
|
name: `${fixtureName} Card`,
|
||||||
slug: "integration-card",
|
slug: `${fixtureName}-card`,
|
||||||
})
|
})
|
||||||
.returning();
|
.returning();
|
||||||
const inventory = await database.db
|
const inventory = await database.db
|
||||||
@@ -229,6 +230,44 @@ test("claims and completes a durable pricing refresh job", async () => {
|
|||||||
assert.equal(user.email, "transactions@example.test");
|
assert.equal(user.email, "transactions@example.test");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("purges completed pricing jobs across sellers before the cutoff", async () => {
|
||||||
|
const first = await createFixture();
|
||||||
|
const second = await createFixture("second");
|
||||||
|
const jobs = await Promise.all(
|
||||||
|
[first, second].map(({ seller, product, condition, inventory }) =>
|
||||||
|
pricingRefreshJobRepository.create({
|
||||||
|
sellerId: seller.id,
|
||||||
|
inventoryItemId: inventory[0].id,
|
||||||
|
productId: product.id,
|
||||||
|
conditionId: condition.id,
|
||||||
|
providerId: "integration",
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const job of jobs) {
|
||||||
|
await pricingRefreshJobRepository.claimNext();
|
||||||
|
await pricingRefreshJobRepository.complete(
|
||||||
|
job.id,
|
||||||
|
[{ priceMinor: 2500 }],
|
||||||
|
{ valueMinor: 2500 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const deleted = await pricingRefreshJobRepository.purgeCompletedBefore(
|
||||||
|
new Date(Date.now() + 1000),
|
||||||
|
);
|
||||||
|
assert.equal(deleted, 2);
|
||||||
|
assert.equal(
|
||||||
|
(await pricingRefreshJobRepository.listForSeller(first.seller.id)).length,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
(await pricingRefreshJobRepository.listForSeller(second.seller.id)).length,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("persists and disables seller-owned provider mappings", async () => {
|
test("persists and disables seller-owned provider mappings", async () => {
|
||||||
const { seller, product } = await createFixture();
|
const { seller, product } = await createFixture();
|
||||||
const created = await providerMappingRepository.createForSeller(
|
const created = await providerMappingRepository.createForSeller(
|
||||||
|
|||||||
@@ -1,3 +1,20 @@
|
|||||||
|
export function getPricingJobRetentionCutoff({ now = new Date(), retentionDays }) {
|
||||||
|
if (!Number.isInteger(retentionDays) || retentionDays < 1)
|
||||||
|
throw new Error("Pricing job retention days must be a positive integer.");
|
||||||
|
const cutoff = new Date(now);
|
||||||
|
cutoff.setUTCDate(cutoff.getUTCDate() - retentionDays);
|
||||||
|
return cutoff;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function purgeCompletedPricingRefreshJobs({
|
||||||
|
jobRepository,
|
||||||
|
now,
|
||||||
|
retentionDays,
|
||||||
|
}) {
|
||||||
|
const cutoff = getPricingJobRetentionCutoff({ now, retentionDays });
|
||||||
|
return jobRepository.purgeCompletedBefore(cutoff);
|
||||||
|
}
|
||||||
|
|
||||||
export async function processNextPersistentPricingRefreshJob({
|
export async function processNextPersistentPricingRefreshJob({
|
||||||
jobRepository,
|
jobRepository,
|
||||||
pricingRefreshJob,
|
pricingRefreshJob,
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import { createHttpPricingProvider } from "../../api/src/modules/pricing/provide
|
|||||||
import { processNextPersistentImageDerivativeJob } from "./image-derivative-worker.js";
|
import { processNextPersistentImageDerivativeJob } from "./image-derivative-worker.js";
|
||||||
import {
|
import {
|
||||||
processNextPersistentPricingRefreshJob,
|
processNextPersistentPricingRefreshJob,
|
||||||
|
purgeCompletedPricingRefreshJobs,
|
||||||
runPricingRefreshJob,
|
runPricingRefreshJob,
|
||||||
} from "./pricing-refresh-worker.js";
|
} from "./pricing-refresh-worker.js";
|
||||||
|
|
||||||
@@ -29,6 +30,14 @@ const pollIntervalMs = Number.parseInt(
|
|||||||
process.env.WORKER_POLL_INTERVAL_MS ?? "2000",
|
process.env.WORKER_POLL_INTERVAL_MS ?? "2000",
|
||||||
10,
|
10,
|
||||||
);
|
);
|
||||||
|
const pricingJobRetentionDays = Number.parseInt(
|
||||||
|
process.env.PRICING_JOB_RETENTION_DAYS ?? "30",
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
const pricingJobCleanupIntervalMs = Number.parseInt(
|
||||||
|
process.env.PRICING_JOB_CLEANUP_INTERVAL_MS ?? "3600000",
|
||||||
|
10,
|
||||||
|
);
|
||||||
|
|
||||||
async function runWorker() {
|
async function runWorker() {
|
||||||
const imageStorageRoot = resolveProjectImageStorageRoot(
|
const imageStorageRoot = resolveProjectImageStorageRoot(
|
||||||
@@ -63,6 +72,15 @@ async function runWorker() {
|
|||||||
renderer: createSharpImageRenderer(),
|
renderer: createSharpImageRenderer(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
async function cleanupPricingJobs() {
|
||||||
|
const deletedCount = await purgeCompletedPricingRefreshJobs({
|
||||||
|
jobRepository: pricingRefreshJobRepository,
|
||||||
|
retentionDays: pricingJobRetentionDays,
|
||||||
|
});
|
||||||
|
if (deletedCount > 0)
|
||||||
|
console.log(`Purged ${deletedCount} completed pricing refresh job(s).`);
|
||||||
|
}
|
||||||
|
|
||||||
async function processOne() {
|
async function processOne() {
|
||||||
const result = await processNextPersistentImageDerivativeJob({
|
const result = await processNextPersistentImageDerivativeJob({
|
||||||
jobRepository,
|
jobRepository,
|
||||||
@@ -94,13 +112,19 @@ async function runWorker() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await processOne();
|
await processOne();
|
||||||
|
await cleanupPricingJobs();
|
||||||
const timer = setInterval(
|
const timer = setInterval(
|
||||||
() => processOne().catch((error) => console.error(error)),
|
() => processOne().catch((error) => console.error(error)),
|
||||||
pollIntervalMs,
|
pollIntervalMs,
|
||||||
);
|
);
|
||||||
|
const cleanupTimer = setInterval(
|
||||||
|
() => cleanupPricingJobs().catch((error) => console.error(error)),
|
||||||
|
pricingJobCleanupIntervalMs,
|
||||||
|
);
|
||||||
|
|
||||||
const shutdown = async () => {
|
const shutdown = async () => {
|
||||||
clearInterval(timer);
|
clearInterval(timer);
|
||||||
|
clearInterval(cleanupTimer);
|
||||||
await database.close();
|
await database.close();
|
||||||
};
|
};
|
||||||
process.once("SIGINT", shutdown);
|
process.once("SIGINT", shutdown);
|
||||||
|
|||||||
@@ -6,10 +6,40 @@ import {
|
|||||||
createPricingProviderRegistry,
|
createPricingProviderRegistry,
|
||||||
} from "../../api/src/modules/pricing/providers/pricing-provider.js";
|
} from "../../api/src/modules/pricing/providers/pricing-provider.js";
|
||||||
import {
|
import {
|
||||||
|
getPricingJobRetentionCutoff,
|
||||||
processNextPersistentPricingRefreshJob,
|
processNextPersistentPricingRefreshJob,
|
||||||
|
purgeCompletedPricingRefreshJobs,
|
||||||
runPricingRefreshJob,
|
runPricingRefreshJob,
|
||||||
} from "../src/pricing-refresh-worker.js";
|
} from "../src/pricing-refresh-worker.js";
|
||||||
|
|
||||||
|
test("calculates a UTC retention cutoff and purges completed jobs before it", async () => {
|
||||||
|
const now = new Date("2026-09-09T12:00:00.000Z");
|
||||||
|
const cutoff = getPricingJobRetentionCutoff({ now, retentionDays: 30 });
|
||||||
|
assert.equal(cutoff.toISOString(), "2026-08-10T12:00:00.000Z");
|
||||||
|
|
||||||
|
let receivedCutoff;
|
||||||
|
const deleted = await purgeCompletedPricingRefreshJobs({
|
||||||
|
jobRepository: {
|
||||||
|
async purgeCompletedBefore(value) {
|
||||||
|
receivedCutoff = value;
|
||||||
|
return 3;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
now,
|
||||||
|
retentionDays: 30,
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(deleted, 3);
|
||||||
|
assert.equal(receivedCutoff.toISOString(), cutoff.toISOString());
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects invalid pricing job retention days", () => {
|
||||||
|
assert.throws(
|
||||||
|
() => getPricingJobRetentionCutoff({ retentionDays: 0 }),
|
||||||
|
/positive integer/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test("completes a pricing refresh through the provider and pricing service boundaries", async () => {
|
test("completes a pricing refresh through the provider and pricing service boundaries", async () => {
|
||||||
const providerRegistry = createPricingProviderRegistry([
|
const providerRegistry = createPricingProviderRegistry([
|
||||||
createPricingProvider({
|
createPricingProvider({
|
||||||
|
|||||||
@@ -125,7 +125,7 @@ Verified locally: `npm test` passes 116 tests with no failures; worker tests, we
|
|||||||
Complete production configuration and operational maturity around the implemented vertical slices.
|
Complete production configuration and operational maturity around the implemented vertical slices.
|
||||||
|
|
||||||
1. Configure and validate an approved external pricing provider and provider-specific mapping data.
|
1. Configure and validate an approved external pricing provider and provider-specific mapping data.
|
||||||
2. Schedule pricing-job retention cleanup and establish restore drills for PostgreSQL and image storage.
|
2. Establish restore drills for PostgreSQL and image storage.
|
||||||
3. Add richer exports/settings, broader accessibility/manual coverage, and any remaining integration edge cases.
|
3. Add richer exports/settings, broader accessibility/manual coverage, and any remaining integration edge cases.
|
||||||
|
|
||||||
Acceptance criteria: production provider credentials are secret-managed, migrations and backups are exercised in the target environment, and CI runs integration, authenticated E2E, web, build, Storybook build, and Storybook interaction/accessibility gates.
|
Acceptance criteria: production provider credentials are secret-managed, migrations and backups are exercised in the target environment, and CI runs integration, authenticated E2E, web, build, Storybook build, and Storybook interaction/accessibility gates.
|
||||||
@@ -136,7 +136,7 @@ Work through one bounded vertical slice at a time. The linked documentation rema
|
|||||||
|
|
||||||
Completed vertical slices include authentication, listings and image prerequisites, acquisitions, sales and audited corrections, dashboard metrics/alerts, pricing observations and quality handling, provider mappings, durable pricing refresh jobs, public search/detail/image states, admin workflows, CI integration/E2E gates, production Docker packaging, rollback/health checks, PostgreSQL backup verification, and deployment smoke checks.
|
Completed vertical slices include authentication, listings and image prerequisites, acquisitions, sales and audited corrections, dashboard metrics/alerts, pricing observations and quality handling, provider mappings, durable pricing refresh jobs, public search/detail/image states, admin workflows, CI integration/E2E gates, production Docker packaging, rollback/health checks, PostgreSQL backup verification, and deployment smoke checks.
|
||||||
|
|
||||||
Remaining roadmap areas are richer exports/settings, approved external provider credentials/adapters, scheduled operational cleanup, broader accessibility/manual coverage, and deployment-specific NAS/TLS validation.
|
Remaining roadmap areas are richer exports/settings, approved external provider credentials/adapters, broader accessibility/manual coverage, and deployment-specific NAS/TLS validation.
|
||||||
|
|
||||||
## Known Gaps
|
## Known Gaps
|
||||||
|
|
||||||
@@ -147,7 +147,7 @@ Remaining roadmap areas are richer exports/settings, approved external provider
|
|||||||
- Product-level provider mappings now persist seller-scoped `PRODUCT`/`SEARCH` relationships and support active listing and controlled disabling.
|
- Product-level provider mappings now persist seller-scoped `PRODUCT`/`SEARCH` relationships and support active listing and controlled disabling.
|
||||||
- Pricing refresh enqueue now resolves its provider from an active Product mapping and rejects unmapped Products explicitly.
|
- Pricing refresh enqueue now resolves its provider from an active Product mapping and rejects unmapped Products explicitly.
|
||||||
- Local development and CI can enable a deterministic fixture provider for end-to-end refresh coverage; production provider adapters remain explicitly configured deployment boundaries.
|
- Local development and CI can enable a deterministic fixture provider for end-to-end refresh coverage; production provider adapters remain explicitly configured deployment boundaries.
|
||||||
- Pricing refresh jobs have a bounded three-attempt retry policy with exponential backoff and an explicit completed-job purge operation for retention scheduling.
|
- Pricing refresh jobs have a bounded three-attempt retry policy with exponential backoff. The worker schedules completed-job cleanup using configurable retention days and cleanup interval settings.
|
||||||
- `npm audit --omit=dev` currently reports zero production vulnerabilities. Re-run the audit after dependency changes and review development-dependency findings separately before production releases.
|
- `npm audit --omit=dev` currently reports zero production vulnerabilities. Re-run the audit after dependency changes and review development-dependency findings separately before production releases.
|
||||||
|
|
||||||
## Local Commands
|
## Local Commands
|
||||||
|
|||||||
+3
-2
@@ -119,8 +119,9 @@ work.
|
|||||||
|
|
||||||
Refresh jobs allow three attempts. Failed attempts requeue with short
|
Refresh jobs allow three attempts. Failed attempts requeue with short
|
||||||
exponential backoff; the third failure becomes terminal `FAILED`. Completed-job
|
exponential backoff; the third failure becomes terminal `FAILED`. Completed-job
|
||||||
retention is handled explicitly through the repository cleanup operation so a
|
retention is handled explicitly through the repository cleanup operation. The
|
||||||
deployment can schedule cleanup according to its retention policy.
|
worker schedules cleanup across all sellers using `PRICING_JOB_RETENTION_DAYS`
|
||||||
|
(default `30`) and `PRICING_JOB_CLEANUP_INTERVAL_MS` (default one hour).
|
||||||
|
|
||||||
Provider failure should not invalidate other valid observations. The failed provider is recorded for admin diagnostics.
|
Provider failure should not invalidate other valid observations. The failed provider is recorded for admin diagnostics.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user