feat: schedule pricing job retention cleanup
Quality / test (push) Canceled after 0s

Schedule configurable completed pricing-job cleanup in the worker, broaden repository cleanup across sellers, add unit and integration coverage, and update pricing operational documentation.
This commit is contained in:
2026-09-09 22:32:04 -04:00
parent 660690bf34
commit 76e40d7289
8 changed files with 126 additions and 14 deletions
+2
View File
@@ -9,6 +9,8 @@ PRICING_FIXTURE_PROVIDER_ENABLED=false
PRICING_HTTP_PROVIDER_ID=http-provider PRICING_HTTP_PROVIDER_ID=http-provider
PRICING_HTTP_PROVIDER_ENDPOINT= PRICING_HTTP_PROVIDER_ENDPOINT=
PRICING_HTTP_PROVIDER_TOKEN= PRICING_HTTP_PROVIDER_TOKEN=
PRICING_JOB_RETENTION_DAYS=30
PRICING_JOB_CLEANUP_INTERVAL_MS=3600000
PASSWORD_RESET_URL=http://localhost:5173/reset-password PASSWORD_RESET_URL=http://localhost:5173/reset-password
SMTP_FROM=no-reply@example.test SMTP_FROM=no-reply@example.test
SMTP_URL=smtp://localhost:1025 SMTP_URL=smtp://localhost:1025
@@ -164,12 +164,11 @@ export function createPricingRefreshJobRepository(db) {
}); });
}, },
async purgeCompletedBefore(sellerId, cutoff) { async purgeCompletedBefore(cutoff) {
const deleted = await db const deleted = await db
.delete(pricingRefreshJobs) .delete(pricingRefreshJobs)
.where( .where(
and( and(
eq(pricingRefreshJobs.sellerId, sellerId),
eq(pricingRefreshJobs.status, "COMPLETED"), eq(pricingRefreshJobs.status, "COMPLETED"),
lte(pricingRefreshJobs.updatedAt, cutoff), lte(pricingRefreshJobs.updatedAt, cutoff),
), ),
@@ -41,11 +41,12 @@ const pricingRefreshJobRepository = createPricingRefreshJobRepository(
const providerMappingRepository = createProviderMappingRepository(database.db); const providerMappingRepository = createProviderMappingRepository(database.db);
const saleRepository = createSaleRepository(database.db); const saleRepository = createSaleRepository(database.db);
async function createFixture() { async function createFixture(suffix = "") {
const fixtureName = suffix ? `transactions-${suffix}` : "transactions";
const [user] = await database.db const [user] = await database.db
.insert(users) .insert(users)
.values({ .values({
email: "transactions@example.test", email: `${fixtureName}@example.test`,
passwordHash: "not-a-real-password", passwordHash: "not-a-real-password",
}) })
.returning(); .returning();
@@ -53,14 +54,14 @@ async function createFixture() {
.insert(sellers) .insert(sellers)
.values({ .values({
userId: user.id, userId: user.id,
displayName: "Transactions Owner", displayName: `${fixtureName} Owner`,
}) })
.returning(); .returning();
const [category] = await database.db const [category] = await database.db
.insert(categories) .insert(categories)
.values({ .values({
name: "Transactions Cards", name: `${fixtureName} Cards`,
slug: "transactions-cards", slug: `${fixtureName}-cards`,
}) })
.returning(); .returning();
const [profile] = await database.db const [profile] = await database.db
@@ -84,8 +85,8 @@ async function createFixture() {
.values({ .values({
sellerId: seller.id, sellerId: seller.id,
categoryId: category.id, categoryId: category.id,
name: "Integration Card", name: `${fixtureName} Card`,
slug: "integration-card", slug: `${fixtureName}-card`,
}) })
.returning(); .returning();
const inventory = await database.db const inventory = await database.db
@@ -229,6 +230,44 @@ test("claims and completes a durable pricing refresh job", async () => {
assert.equal(user.email, "transactions@example.test"); assert.equal(user.email, "transactions@example.test");
}); });
test("purges completed pricing jobs across sellers before the cutoff", async () => {
const first = await createFixture();
const second = await createFixture("second");
const jobs = await Promise.all(
[first, second].map(({ seller, product, condition, inventory }) =>
pricingRefreshJobRepository.create({
sellerId: seller.id,
inventoryItemId: inventory[0].id,
productId: product.id,
conditionId: condition.id,
providerId: "integration",
}),
),
);
for (const job of jobs) {
await pricingRefreshJobRepository.claimNext();
await pricingRefreshJobRepository.complete(
job.id,
[{ priceMinor: 2500 }],
{ valueMinor: 2500 },
);
}
const deleted = await pricingRefreshJobRepository.purgeCompletedBefore(
new Date(Date.now() + 1000),
);
assert.equal(deleted, 2);
assert.equal(
(await pricingRefreshJobRepository.listForSeller(first.seller.id)).length,
0,
);
assert.equal(
(await pricingRefreshJobRepository.listForSeller(second.seller.id)).length,
0,
);
});
test("persists and disables seller-owned provider mappings", async () => { test("persists and disables seller-owned provider mappings", async () => {
const { seller, product } = await createFixture(); const { seller, product } = await createFixture();
const created = await providerMappingRepository.createForSeller( const created = await providerMappingRepository.createForSeller(
+17
View File
@@ -1,3 +1,20 @@
export function getPricingJobRetentionCutoff({ now = new Date(), retentionDays }) {
if (!Number.isInteger(retentionDays) || retentionDays < 1)
throw new Error("Pricing job retention days must be a positive integer.");
const cutoff = new Date(now);
cutoff.setUTCDate(cutoff.getUTCDate() - retentionDays);
return cutoff;
}
export async function purgeCompletedPricingRefreshJobs({
jobRepository,
now,
retentionDays,
}) {
const cutoff = getPricingJobRetentionCutoff({ now, retentionDays });
return jobRepository.purgeCompletedBefore(cutoff);
}
export async function processNextPersistentPricingRefreshJob({ export async function processNextPersistentPricingRefreshJob({
jobRepository, jobRepository,
pricingRefreshJob, pricingRefreshJob,
+24
View File
@@ -22,6 +22,7 @@ import { createHttpPricingProvider } from "../../api/src/modules/pricing/provide
import { processNextPersistentImageDerivativeJob } from "./image-derivative-worker.js"; import { processNextPersistentImageDerivativeJob } from "./image-derivative-worker.js";
import { import {
processNextPersistentPricingRefreshJob, processNextPersistentPricingRefreshJob,
purgeCompletedPricingRefreshJobs,
runPricingRefreshJob, runPricingRefreshJob,
} from "./pricing-refresh-worker.js"; } from "./pricing-refresh-worker.js";
@@ -29,6 +30,14 @@ const pollIntervalMs = Number.parseInt(
process.env.WORKER_POLL_INTERVAL_MS ?? "2000", process.env.WORKER_POLL_INTERVAL_MS ?? "2000",
10, 10,
); );
const pricingJobRetentionDays = Number.parseInt(
process.env.PRICING_JOB_RETENTION_DAYS ?? "30",
10,
);
const pricingJobCleanupIntervalMs = Number.parseInt(
process.env.PRICING_JOB_CLEANUP_INTERVAL_MS ?? "3600000",
10,
);
async function runWorker() { async function runWorker() {
const imageStorageRoot = resolveProjectImageStorageRoot( const imageStorageRoot = resolveProjectImageStorageRoot(
@@ -63,6 +72,15 @@ async function runWorker() {
renderer: createSharpImageRenderer(), renderer: createSharpImageRenderer(),
}); });
async function cleanupPricingJobs() {
const deletedCount = await purgeCompletedPricingRefreshJobs({
jobRepository: pricingRefreshJobRepository,
retentionDays: pricingJobRetentionDays,
});
if (deletedCount > 0)
console.log(`Purged ${deletedCount} completed pricing refresh job(s).`);
}
async function processOne() { async function processOne() {
const result = await processNextPersistentImageDerivativeJob({ const result = await processNextPersistentImageDerivativeJob({
jobRepository, jobRepository,
@@ -94,13 +112,19 @@ async function runWorker() {
} }
await processOne(); await processOne();
await cleanupPricingJobs();
const timer = setInterval( const timer = setInterval(
() => processOne().catch((error) => console.error(error)), () => processOne().catch((error) => console.error(error)),
pollIntervalMs, pollIntervalMs,
); );
const cleanupTimer = setInterval(
() => cleanupPricingJobs().catch((error) => console.error(error)),
pricingJobCleanupIntervalMs,
);
const shutdown = async () => { const shutdown = async () => {
clearInterval(timer); clearInterval(timer);
clearInterval(cleanupTimer);
await database.close(); await database.close();
}; };
process.once("SIGINT", shutdown); process.once("SIGINT", shutdown);
@@ -6,10 +6,40 @@ import {
createPricingProviderRegistry, createPricingProviderRegistry,
} from "../../api/src/modules/pricing/providers/pricing-provider.js"; } from "../../api/src/modules/pricing/providers/pricing-provider.js";
import { import {
getPricingJobRetentionCutoff,
processNextPersistentPricingRefreshJob, processNextPersistentPricingRefreshJob,
purgeCompletedPricingRefreshJobs,
runPricingRefreshJob, runPricingRefreshJob,
} from "../src/pricing-refresh-worker.js"; } from "../src/pricing-refresh-worker.js";
test("calculates a UTC retention cutoff and purges completed jobs before it", async () => {
const now = new Date("2026-09-09T12:00:00.000Z");
const cutoff = getPricingJobRetentionCutoff({ now, retentionDays: 30 });
assert.equal(cutoff.toISOString(), "2026-08-10T12:00:00.000Z");
let receivedCutoff;
const deleted = await purgeCompletedPricingRefreshJobs({
jobRepository: {
async purgeCompletedBefore(value) {
receivedCutoff = value;
return 3;
},
},
now,
retentionDays: 30,
});
assert.equal(deleted, 3);
assert.equal(receivedCutoff.toISOString(), cutoff.toISOString());
});
test("rejects invalid pricing job retention days", () => {
assert.throws(
() => getPricingJobRetentionCutoff({ retentionDays: 0 }),
/positive integer/,
);
});
test("completes a pricing refresh through the provider and pricing service boundaries", async () => { test("completes a pricing refresh through the provider and pricing service boundaries", async () => {
const providerRegistry = createPricingProviderRegistry([ const providerRegistry = createPricingProviderRegistry([
createPricingProvider({ createPricingProvider({
+3 -3
View File
@@ -125,7 +125,7 @@ Verified locally: `npm test` passes 116 tests with no failures; worker tests, we
Complete production configuration and operational maturity around the implemented vertical slices. Complete production configuration and operational maturity around the implemented vertical slices.
1. Configure and validate an approved external pricing provider and provider-specific mapping data. 1. Configure and validate an approved external pricing provider and provider-specific mapping data.
2. Schedule pricing-job retention cleanup and establish restore drills for PostgreSQL and image storage. 2. Establish restore drills for PostgreSQL and image storage.
3. Add richer exports/settings, broader accessibility/manual coverage, and any remaining integration edge cases. 3. Add richer exports/settings, broader accessibility/manual coverage, and any remaining integration edge cases.
Acceptance criteria: production provider credentials are secret-managed, migrations and backups are exercised in the target environment, and CI runs integration, authenticated E2E, web, build, Storybook build, and Storybook interaction/accessibility gates. Acceptance criteria: production provider credentials are secret-managed, migrations and backups are exercised in the target environment, and CI runs integration, authenticated E2E, web, build, Storybook build, and Storybook interaction/accessibility gates.
@@ -136,7 +136,7 @@ Work through one bounded vertical slice at a time. The linked documentation rema
Completed vertical slices include authentication, listings and image prerequisites, acquisitions, sales and audited corrections, dashboard metrics/alerts, pricing observations and quality handling, provider mappings, durable pricing refresh jobs, public search/detail/image states, admin workflows, CI integration/E2E gates, production Docker packaging, rollback/health checks, PostgreSQL backup verification, and deployment smoke checks. Completed vertical slices include authentication, listings and image prerequisites, acquisitions, sales and audited corrections, dashboard metrics/alerts, pricing observations and quality handling, provider mappings, durable pricing refresh jobs, public search/detail/image states, admin workflows, CI integration/E2E gates, production Docker packaging, rollback/health checks, PostgreSQL backup verification, and deployment smoke checks.
Remaining roadmap areas are richer exports/settings, approved external provider credentials/adapters, scheduled operational cleanup, broader accessibility/manual coverage, and deployment-specific NAS/TLS validation. Remaining roadmap areas are richer exports/settings, approved external provider credentials/adapters, broader accessibility/manual coverage, and deployment-specific NAS/TLS validation.
## Known Gaps ## Known Gaps
@@ -147,7 +147,7 @@ Remaining roadmap areas are richer exports/settings, approved external provider
- Product-level provider mappings now persist seller-scoped `PRODUCT`/`SEARCH` relationships and support active listing and controlled disabling. - Product-level provider mappings now persist seller-scoped `PRODUCT`/`SEARCH` relationships and support active listing and controlled disabling.
- Pricing refresh enqueue now resolves its provider from an active Product mapping and rejects unmapped Products explicitly. - Pricing refresh enqueue now resolves its provider from an active Product mapping and rejects unmapped Products explicitly.
- Local development and CI can enable a deterministic fixture provider for end-to-end refresh coverage; production provider adapters remain explicitly configured deployment boundaries. - Local development and CI can enable a deterministic fixture provider for end-to-end refresh coverage; production provider adapters remain explicitly configured deployment boundaries.
- Pricing refresh jobs have a bounded three-attempt retry policy with exponential backoff and an explicit completed-job purge operation for retention scheduling. - Pricing refresh jobs have a bounded three-attempt retry policy with exponential backoff. The worker schedules completed-job cleanup using configurable retention days and cleanup interval settings.
- `npm audit --omit=dev` currently reports zero production vulnerabilities. Re-run the audit after dependency changes and review development-dependency findings separately before production releases. - `npm audit --omit=dev` currently reports zero production vulnerabilities. Re-run the audit after dependency changes and review development-dependency findings separately before production releases.
## Local Commands ## Local Commands
+3 -2
View File
@@ -119,8 +119,9 @@ work.
Refresh jobs allow three attempts. Failed attempts requeue with short Refresh jobs allow three attempts. Failed attempts requeue with short
exponential backoff; the third failure becomes terminal `FAILED`. Completed-job exponential backoff; the third failure becomes terminal `FAILED`. Completed-job
retention is handled explicitly through the repository cleanup operation so a retention is handled explicitly through the repository cleanup operation. The
deployment can schedule cleanup according to its retention policy. worker schedules cleanup across all sellers using `PRICING_JOB_RETENTION_DAYS`
(default `30`) and `PRICING_JOB_CLEANUP_INTERVAL_MS` (default one hour).
Provider failure should not invalidate other valid observations. The failed provider is recorded for admin diagnostics. Provider failure should not invalidate other valid observations. The failed provider is recorded for admin diagnostics.