fix: preserve player assets across Grand Exchange recovery

This commit is contained in:
max bittker committed 2026-09-07 23:47:29 -04:00
1 parent 69d5b709c8
commit e5282efdd9
9 files changed
+312 -42

No files matched your search

+20 -6
View File
@@ -52,12 +52,26 @@ in-memory backpack. Counterparties' inventory is never remotely modified:
only their escrow claims change. Cancellation is an idempotent ledger-only
transaction.
Placement and collection also checkpoint online players. Normal saves update
existing checkpoints, and player loading prefers them over login-service saves.
This makes the ledger authoritative for bystanders as well as traders. Recovery
must account for temporary inventories and pending logout saves; checkpointing
online players alone is not sufficient. Transient NPC, ground-item and shop state
is not persisted by the exchange.
Placement and collection checkpoint online players and pending logout saves in
the same transaction. Autosaves, reconnects and logouts use the same world-wide
checkpoint boundary, including first-time players. Loading prefers ledger saves
over login-service saves, making the ledger authoritative for bystanders as well
as traders. Saving one participant after an ordinary trade cannot advance only
that participant's recovery state.
Save version 8 includes player-owned temporary escrow (trade, duel, party-chest,
death-kept items and unclaimed rewards). Taking a snapshot never closes another
player's interface or changes their live inventory. On recovery these items
return to the backpack, then the bank (notes are unnoted there). Stack/capacity
overflow remains saved and is retried while the player is idle. Display-only
crafting inventories are excluded. Version 7 saves remain readable; an older
server cannot read version 8 saves, so rollback requires a compatible build.
Transient NPC, ground-item and shop state is not persisted by the exchange.
Collect-all transfers all six slots in one transaction and checkpoints the world
once. Empty collections and collections blocked entirely by inventory capacity
do not serialize or checkpoint players. A failed batch restores all its claims
and inventory additions.
**Operational consequences:** back up the ledger and its player checkpoints
with the rest of the world. Use SQLite's backup API or stop the world before
+1 -1
View File
@@ -9,7 +9,7 @@
"test": "bun test sdk/test server/webclient/src/bot server/webclient/src/util server/webclient/src/lite server/engine/test/market.test.ts server/engine/test/market-web.test.ts server/engine/test/item-search.test.ts server/engine/test/file-stream.test.ts",
"typecheck": "tsc --noEmit",
"typecheck:webclient": "tsc --noEmit -p server/webclient/tsconfig.json",
"test:market:integration": "cd server/engine && bun test/market-integration.ts && bun test/market-protocol.ts && bun test/market-disabled.ts"
"test:market:integration": "cd server/engine && bun test/market-integration.ts && bun test/market-recovery.ts && bun test/market-protocol.ts && bun test/market-disabled.ts"
},
"devDependencies": {
"@types/bun": "latest",
+26 -3
View File
@@ -1,4 +1,5 @@
import { tickExchange } from '#/engine/market/GrandExchange.js';
import { playerSaveStore } from '#/engine/market/MarketStore.js';
// stdlib
import fs from 'fs';
import { Worker } from 'worker_threads';
@@ -795,6 +796,7 @@ class World {
// - interactions
// - movement
player.processInteraction();
if (!player.busy() && !player.delayed && !player.loggingOut) player.recoverItems();
if (Environment.GE_ENABLED) tickExchange(player);
// - run energy
@@ -1401,15 +1403,34 @@ class World {
return;
}
const saves = this.checkpointPlayers();
for (const player of this.playerLoop.all()) {
this.loginThread.postMessage({
type: 'player_autosave',
username: player.username,
save: player.save()
save: saves.get(player.username)!
});
}
}
/** A single recovery boundary includes online players and pending logout saves.
* Used by autosaves, reconnects, logouts and GE so subsequent ordinary trades
* cannot advance only one side of a previously checkpointed transfer.
*/
checkpointPlayers(extra?: Player, extraSave?: Uint8Array): Map<string, Uint8Array> {
const saves = new Map<string, Uint8Array>();
for (const [owner, request] of this.logoutRequests) saves.set(owner, request.save);
for (const player of this.playerLoop.all()) {
saves.set(player.username, player === extra && extraSave ? extraSave : player.save(false));
}
if (extra) saves.set(extra.username, extraSave ?? extra.save(false));
const store = playerSaveStore();
if (store) store.db.transaction(() => {
for (const [owner, save] of saves) store.checkpoint(owner, save, true);
}).immediate();
return saves;
}
enqueueScript(script: ScriptState, delay: number = 0): void {
this.queue.addTail(new EntityQueueState(script, delay + 1));
}
@@ -2727,12 +2748,14 @@ class World {
}
flushPlayer(player: Player) {
const save = player.save();
const save = player.save(false);
this.logoutRequests.set(player.username, {
save,
lastAttempt: -1
});
// Enroll first-time players before the async login-service acknowledgement.
// Keep the pending save available even if a bystander's serialization fails.
this.checkpointPlayers(player, save);
}
}
+35 -2
View File
@@ -188,7 +188,26 @@ export default class Player extends PathingEntity {
[81, 44]
]);
save() {
// Only containers holding player-owned assets belong here; crafting previews do not.
static readonly RECOVERABLE_INVENTORIES = ['tradeoffer', 'dueloffer', 'duelwinnings', 'partyroom_tempinv', 'deathkeep', 'trail_rewardinv', 'trawler_rewardinv', 'duelarrows'];
recoveryItems: { id: number; count: number }[] = [];
/** Retry saved overflow when space becomes available, without dropping anything. */
recoverItems() {
if (!this.recoveryItems.length) return;
const inv = this.getInventory(InvType.INV)!;
const bank = this.getInventory(InvType.getId('bank'))!;
this.recoveryItems = this.recoveryItems.flatMap(item => {
let count = item.count - inv.add(item.id, item.count);
if (count) {
const type = ObjType.get(item.id);
count -= bank.add(type.certtemplate !== -1 ? type.certlink : item.id, count);
}
return count ? [{ id: item.id, count }] : [];
});
}
save(checkpoint = true) {
const sav = Packet.alloc(2);
sav.p2(PlayerLoading.SAV_MAGIC); // magic
sav.p2(PlayerLoading.SAV_VERSION); // version
@@ -271,9 +290,23 @@ export default class Player extends PathingEntity {
// last login info
sav.p8(this.lastLoginTime);
// Serialize escrow separately. Never close another player's interface or mutate
// their live inventory while taking an exchange transaction's recovery snapshot.
const recovery = [...this.recoveryItems];
for (const name of Player.RECOVERABLE_INVENTORIES) {
const inventory = this.invs.get(InvType.getId(name));
if (inventory) recovery.push(...inventory.itemsFiltered);
}
if (recovery.length > 65535) throw new Error('Too many recovery items to save.');
sav.p2(recovery.length);
for (const item of recovery) {
sav.p2(item.id);
sav.p4(item.count);
}
sav.p4(Packet.getcrc(sav.data, 0, sav.pos));
const bytes = sav.data.subarray(0, sav.pos);
playerSaveStore()?.checkpoint(this.username, bytes);
if (checkpoint && playerSaveStore()) World.checkpointPlayers(this, bytes);
return bytes;
}
@@ -10,7 +10,7 @@ import { fromBase37, toBase37 } from '#/util/JString.js';
export class PlayerLoading {
public static readonly SAV_MAGIC: number = 0x2004;
public static readonly SAV_VERSION: number = 7;
public static readonly SAV_VERSION: number = 8;
static verify(sav: Packet) {
if (sav.g2() !== PlayerLoading.SAV_MAGIC) {
@@ -162,6 +162,14 @@ export class PlayerLoading {
player.lastLoginTime = sav.g8();
}
if (version >= 8) {
const count = sav.g2();
for (let i = 0; i < count; i++) {
player.recoveryItems.push({ id: sav.g2(), count: sav.g4s() });
}
player.recoverItems();
}
player.combatLevel = player.getCombatLevel();
return player;
@@ -123,10 +123,9 @@ function account(p: Player): MarketAccount {
// One durable cut across carried player inventories prevents an item handed
// from A to B between autosaves reappearing in A after B deposits it here.
// This runs inside the exchange transaction, without yielding the world.
for (const other of World.playerLoop.all()) {
if (other !== p) marketStore().checkpoint(other.username, other.save(), true);
}
return p.save();
const save = p.save(false);
World.checkpointPlayers(p, save);
return save;
},
rollback: () => {
before.forEach((item, i) => inv.set(i, item));
@@ -420,20 +419,10 @@ export function exchangeButton(p: Player, id: number): boolean {
}
}
if (id === com('collect')) {
let items = 0,
gold = 0;
// Each offer is a separate atomic collection. Preserve already committed
// additions in the receipt if a later offer fails (e.g. a save error).
s.receipt = { token: ++nextReceipt, kind: 'collect', items: 0, coins: 0 };
for (const offer of marketStore().offers(p.username)) {
const result = marketStore().collect(account(p), offer.id, coins(), noteFor(offer.item));
items += result.items;
gold += result.coins;
s.receipt.items = items;
s.receipt.coins = gold;
}
const result = marketStore().collectMany(account(p), marketStore().offers(p.username).map(offer => ({ id: offer.id, itemId: noteFor(offer.item) })), coins());
s.receipt = { token: ++nextReceipt, kind: 'collect', ...result };
s.screen = 'home';
render(p, `Collected ${items} items and ${gold} coins. Any overflow stays here.`);
render(p, `Collected ${result.items} items and ${result.coins} coins. Any overflow stays here.`);
return true;
}
if (id === com('close')) {
+24 -12
View File
@@ -100,13 +100,13 @@ export class MarketStore {
saved(owner: string): Uint8Array | undefined {
return this.db.query<{ save: Uint8Array }, [string]>('SELECT save FROM accounts WHERE owner=?').get(owner)?.save;
}
private transfer<T>(account: MarketAccount, action: () => T): T {
private transfer<T>(account: MarketAccount, action: () => T, needsCheckpoint = () => true): T {
try {
let value!: T;
this.db
.transaction(() => {
const result = action();
this.checkpoint(account.owner, account.save(), true);
if (needsCheckpoint()) this.checkpoint(account.owner, account.save(), true);
value = result;
})
.immediate();
@@ -184,17 +184,29 @@ export class MarketStore {
.immediate();
}
collect(account: MarketAccount, id: number, coinsId: number, itemId?: number): { items: number; coins: number } {
return this.collectMany(account, [{ id, itemId }], coinsId);
}
/** One inventory snapshot and durable transaction for all requested claims. */
collectMany(account: MarketAccount, requests: readonly { id: number; itemId?: number }[], coinsId: number): { items: number; coins: number } {
if (new Set(requests.map(r => r.id)).size !== requests.length) throw new Error('Duplicate collection offer.');
// Synchronous single-world access: no action can interleave with this preflight.
const offers = requests.map(request => ({ ...request, offer: this.owned(account.owner, request.id) }));
const result = { items: 0, coins: 0 };
if (offers.every(({ offer }) => !offer.items && !offer.coins && offer.state === 'open')) return result;
return this.transfer(account, () => {
const o = this.owned(account.owner, id);
const items = o.items ? account.give(itemId ?? o.item, o.items) : 0;
const coins = o.coins ? account.give(coinsId, o.coins) : 0;
if (!Number.isInteger(items) || items < 0 || items > o.items || !Number.isInteger(coins) || coins < 0 || coins > o.coins) throw new Error('Invalid inventory transfer.');
o.items -= items;
o.coins -= coins;
if (o.state !== 'open' && !o.items && !o.coins) o.slot = -1;
this.update(o);
return { items, coins };
});
for (const { offer: o, itemId } of offers) {
const items = o.items ? account.give(itemId ?? o.item, o.items) : 0;
const coins = o.coins ? account.give(coinsId, o.coins) : 0;
if (!Number.isInteger(items) || items < 0 || items > o.items || !Number.isInteger(coins) || coins < 0 || coins > o.coins) throw new Error('Invalid inventory transfer.');
o.items -= items;
o.coins -= coins;
if (o.state !== 'open' && !o.items && !o.coins) o.slot = -1;
if (items || coins || o.slot === -1) this.update(o);
result.items += items;
result.coins += coins;
}
return result;
}, () => result.items > 0 || result.coins > 0);
}
quote(item: number, since = Date.now() - 86_400_000) {
const book = this.db
+163
View File
@@ -0,0 +1,163 @@
// Real saves and native handlers, with only disposable players and a temporary ledger.
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { strict as assert } from 'node:assert';
const dir = mkdtempSync(join(tmpdir(), 'ge-recovery-'));
process.env.GE_DATABASE = join(dir, 'market.sqlite');
process.env.GE_ENABLED = 'true';
const { default: World } = await import('../src/engine/World.js');
const { default: Player } = await import('../src/engine/entity/Player.js');
const { default: Component } = await import('../src/cache/config/Component.js');
const { default: ObjType } = await import('../src/cache/config/ObjType.js');
const { default: InvType } = await import('../src/cache/config/InvType.js');
const { default: Packet } = await import('../src/io/Packet.js');
const { PlayerLoading } = await import('../src/engine/entity/PlayerLoading.js');
const { default: IfButtonHandler } = await import('../src/network/game/client/handler/IfButtonHandler.js');
const { default: IfButton } = await import('../src/network/game/client/model/IfButton.js');
const { openExchange, exchangeSearch, exchangeCount } = await import('../src/engine/market/GrandExchange.js');
const { marketStore, MAX_GP } = await import('../src/engine/market/MarketStore.js');
const { noteFor } = await import('../src/engine/market/MarketCatalog.js');
try {
World.reload();
const inv = InvType.getId('inv'), bank = InvType.getId('bank'), trade = InvType.getId('tradeoffer');
const coins = ObjType.getId('coins'), logs = ObjType.getId('logs'), note = noteFor(logs);
let key = 100n;
const make = (name: string, balance = 0) => {
const p = PlayerLoading.load(name, new Packet(new Uint8Array()), null);
p.x = 3181; p.z = 3443; p.level = 0;
World.playerLoop.add(key++, p);
if (balance) p.invAdd(inv, coins, balance);
return p;
};
const reload = (p: InstanceType<typeof Player>, previous = new Uint8Array()) => PlayerLoading.load(p.username, new Packet(previous), null);
const click = (p: InstanceType<typeof Player>, name: string) => assert(new IfButtonHandler().handle(new IfButton(Component.getId('grand_exchange:' + name)), p));
const place = (p: InstanceType<typeof Player>, slot = 0) => {
openExchange(p, 3180, 3443);
click(p, 'slot' + slot + '_buy');
assert(exchangeSearch(p, 'logs'));
assert(exchangeSearch(p, 'logs', logs));
click(p, 'offer_price');
assert(exchangeCount(p, 1));
click(p, 'offer_confirm');
};
const actor = make('recoveractor', 100);
const victims = Player.RECOVERABLE_INVENTORIES.map((name, i) => {
const p = make('recover' + i);
p.invAdd(inv, note, 100);
const previous = p.save(false);
p.getInventory(inv)!.remove(note, 100);
p.invAdd(InvType.getId(name), note, 100);
p.openMainModal(Component.getId('trademain'));
return { p, previous, name };
});
place(actor);
for (const { p, previous, name } of victims) {
assert.equal(p.modalMain, Component.getId('trademain'), 'Checkpoint leaves the live interface open');
assert.equal(p.getInventory(InvType.getId(name))!.getItemCount(note), 100, 'Checkpoint leaves live escrow unchanged');
const restored = reload(p, previous);
assert.equal(restored.getInventory(inv)!.getItemCount(note), 100, name + ' survives recovery');
assert.equal(restored.getInventory(InvType.getId(name))!.getItemCount(note), 0, 'Escrow is not restored twice');
assert(PlayerLoading.verify(new Packet(restored.save(false))));
}
console.log('PASS: temporary owned inventories recover without changing live trades.');
// Completing a checkpointed trade, then saving only its recipient, must not
// leave the donor's earlier recovery escrow available for a second recovery.
const donor = victims[0].p;
donor.getInventory(trade)!.remove(note, 100);
actor.invAdd(inv, note, 100);
actor.save(); // reconnect/save path
assert.equal(reload(donor).getInventory(inv)!.getItemCount(note), 0);
assert.equal(reload(actor).getInventory(inv)!.getItemCount(note), 100);
for (const legacy of [false, true]) {
const giver = make(legacy ? 'pendingold' : 'pendingnew', 100);
const receiver = make(legacy ? 'receiverold' : 'receivernew');
const previous = giver.save(false);
giver.getInventory(inv)!.remove(coins, 100);
receiver.invAdd(inv, coins, 100);
giver.unlink();
if (legacy) World.logoutRequests.set(giver.username, { save: giver.save(false), lastAttempt: -1 });
else {
World.flushPlayer(giver);
assert(marketStore().saved(giver.username), 'First logout is durable before the login-service ack');
}
place(receiver);
const total = reload(giver, previous).getInventory(inv)!.getItemCount(coins)
+ reload(receiver).getInventory(inv)!.getItemCount(coins)
+ marketStore().offers(receiver.username).reduce((n, o) => n + o.remaining * o.price, 0);
assert.equal(total, 100, 'Pending logout donor cannot recover transferred coins twice');
}
console.log('PASS: completed transfers and pending logout saves recover exactly once.');
// Both coin stacks are full. Overflow must survive another save/load and retry.
const full = make('recoverfull', MAX_GP);
full.invAdd(bank, coins, MAX_GP);
full.invAdd(trade, coins, 100);
full.save();
let overflow = reload(full);
full.unlink();
assert.deepEqual(overflow.recoveryItems, [{ id: coins, count: 100 }]);
overflow.save();
overflow = reload(overflow);
assert.deepEqual(overflow.recoveryItems, [{ id: coins, count: 100 }]);
overflow.getInventory(inv)!.remove(coins, 50);
overflow.recoverItems();
assert.equal(overflow.getInventory(inv)!.getItemCount(coins), MAX_GP);
assert.deepEqual(overflow.recoveryItems, [{ id: coins, count: 50 }]);
overflow.getInventory(bank)!.remove(coins, 50);
overflow.recoverItems();
assert.equal(overflow.getInventory(bank)!.getItemCount(coins), MAX_GP);
assert.deepEqual(overflow.recoveryItems, []);
overflow.save();
assert.deepEqual(reload(overflow).recoveryItems, []);
const noted = make('recovernotes');
for (let slot = 0; slot < 28; slot++) noted.getInventory(inv)!.set(slot, { id: logs, count: 1 });
noted.invAdd(trade, note, 100);
noted.save();
assert.equal(reload(noted).getInventory(bank)!.getItemCount(logs), 100, 'Recovered notes are unnoted when banked');
assert.equal(reload(noted).getInventory(bank)!.getItemCount(note), 0);
console.log('PASS: stack overflow remains durable and bank recovery handles notes.');
for (let slot = 1; slot < 6; slot++) place(actor, slot);
const observer = make('saveobserver');
let saves = 0;
const originalSave = observer.save.bind(observer);
observer.save = checkpoint => { saves++; return originalSave(checkpoint); };
for (let i = 0; i < 5; i++) click(actor, 'collect');
assert.equal(saves, 0, 'Empty collections perform no world saves');
for (const offer of marketStore().offers(actor.username)) marketStore().cancel(actor.username, offer.id);
click(actor, 'collect');
assert.equal(saves, 1, 'Six actual collections checkpoint each bystander only once');
assert.equal(marketStore().offers(actor.username).length, 0);
// A failure after one bystander was serialized cannot commit any checkpoint.
const beforeActor = actor.getInventory(inv)!.getItemCount(coins);
const beforeObserver = marketStore().saved(observer.username)!.slice();
const broken = make('brokensave');
broken.save = () => { throw new Error('Injected serialization failure'); };
place(actor);
assert.equal(actor.getInventory(inv)!.getItemCount(coins), beforeActor);
assert.equal(marketStore().offers(actor.username).length, 0);
assert.deepEqual(marketStore().saved(observer.username), beforeObserver);
broken.unlink();
console.log('PASS: empty collections, one-checkpoint batching and failed saves are safe.');
// Version 7 remains readable; create a valid older-format save from an empty
// version 8 save by removing its empty recovery section and updating its CRC.
const older = make('oldversion', 42).save(false);
const oldBytes = new Uint8Array(older.length - 2);
oldBytes.set(older.subarray(0, older.length - 6));
const oldPacket = new Packet(oldBytes);
oldPacket.pos = 2; oldPacket.p2(7);
oldPacket.pos = oldBytes.length - 4;
oldPacket.p4(Packet.getcrc(oldBytes, 0, oldPacket.pos));
assert.equal(PlayerLoading.load('oldversion2', new Packet(oldBytes), null).getInventory(inv)!.getItemCount(coins), 42);
console.log('PASS: existing version 7 saves remain readable.');
} finally {
marketStore().db.close();
rmSync(dir, { recursive: true, force: true });
await Promise.all([World.loginThread.terminate(), World.friendThread.terminate(), World.loggerThread.terminate()]);
}
+28
View File
@@ -149,6 +149,34 @@ describe('Grand Exchange conservation and matching', () => {
expect(() => s.cancel('b', o.id)).toThrow();
expect(() => s.collect(b.account(), o.id, COINS)).toThrow();
});
test('empty and capacity-blocked collections never serialize the world', () => {
const s = store(), a = new Wallet('a', 100);
const o = s.place(a.account(), ITEM, 'buy', 1, 10, COINS);
a.failSave = true;
expect(s.collect(a.account(), o.id, COINS)).toEqual({ items: 0, coins: 0 });
s.cancel(a.owner, o.id);
a.capacity = 90;
expect(s.collect(a.account(), o.id, COINS)).toEqual({ items: 0, coins: 0 });
expect(s.offers(a.owner)[0]?.coins).toBe(10);
});
test('batch collection validates every owner and rolls all offers back on save failure', () => {
const s = store(), a = new Wallet('a', 100), b = new Wallet('b', 100);
const first = s.place(a.account(), ITEM, 'buy', 1, 10, COINS);
const second = s.place(a.account(), ITEM, 'buy', 1, 20, COINS);
const foreign = s.place(b.account(), ITEM, 'buy', 1, 30, COINS);
for (const o of [first, second, foreign]) s.cancel(o.owner, o.id);
expect(() => s.collectMany(a.account(), [{ id: first.id }, { id: foreign.id }], COINS)).toThrow('Offer not found');
expect(() => s.collectMany(a.account(), [{ id: first.id }, { id: first.id }], COINS)).toThrow('Duplicate');
a.failSave = true;
const requests = [{ id: first.id }, { id: second.id }];
expect(() => s.collectMany(a.account(), requests, COINS)).toThrow('Disk failure');
expect(a.inv[COINS]).toBe(70);
expect(s.offers(a.owner).map(o => o.coins)).toEqual([10, 20]);
a.failSave = false;
expect(s.collectMany(a.account(), requests, COINS)).toEqual({ items: 0, coins: 30 });
expect(a.inv[COINS]).toBe(100);
expect(s.offers(a.owner)).toEqual([]);
});
test('invalid totals and insufficient assets do not change ledger or inventory', () => {
const s = store(),
a = new Wallet('a', 100);