From 9963ab79b7c7392d5f87c98cd55953ffa004efd9 Mon Sep 17 00:00:00 2001 From: Julien Dessaux Date: Wed, 24 May 2023 23:03:19 +0200 Subject: Rewrote the api rate limiter with promises instead of callbacks --- database/001_systems.sql | 6 +++++ database/db.js | 1 + database/systems.js | 26 +++++++++++++++++++ lib/api.js | 67 ++++++++++++++++++++++-------------------------- lib/systems.js | 36 ++++++++++++++++++++++++++ main.js | 9 ++++--- 6 files changed, 105 insertions(+), 40 deletions(-) create mode 100644 database/001_systems.sql create mode 100644 database/systems.js create mode 100644 lib/systems.js diff --git a/database/001_systems.sql b/database/001_systems.sql new file mode 100644 index 0000000..8448955 --- /dev/null +++ b/database/001_systems.sql @@ -0,0 +1,6 @@ +CREATE TABLE systems ( + id INTEGER PRIMARY KEY, + symbol TEXT NOT NULL UNIQUE, + data TEXT NOT NULL, + updated DATE DEFAULT (datetime('now')) +); diff --git a/database/db.js b/database/db.js index db71080..6deddcc 100644 --- a/database/db.js +++ b/database/db.js @@ -3,6 +3,7 @@ import Database from 'better-sqlite3'; const allMigrations = [ 'database/000_init.sql', + 'database/001_systems.sql', ]; const db = new Database( diff --git a/database/systems.js b/database/systems.js new file mode 100644 index 0000000..46b4663 --- /dev/null +++ b/database/systems.js @@ -0,0 +1,26 @@ +import db from './db.js'; + +const getSystemStatement = db.prepare(`SELECT data from systems where symbol = ?;`); +const setSystemStatement = db.prepare(`INSERT INTO systems(symbol, data) VALUES (?, ?);`); + +export function getSystem(symbol) { + try { + const data = getSystemStatement.get(symbol); + if (data === undefined) { + return null; + } + return JSON.parse(data.data); + } catch (err) { + console.log(err); + return null; + } +} + +export function setSystem(symbol, data) { + try { + return setSystemStatement.run(symbol, JSON.stringify(data)).lastInsertRowid; + } catch (err) { + console.log(err); + return null; + } +} diff --git a/lib/api.js b/lib/api.js index b1cdcb8..bbb2977 100644 --- a/lib/api.js +++ b/lib/api.js @@ -5,32 +5,28 @@ let busy = false; // lets us know if we are already sending api requests or not. let headers = undefined; // a file scope variable so that we only evaluate these once. let queue = new PriorityQueue(); // a priority queue to hold api calls we want to send, allows for throttling. -// chain takes an array of actions as argument. For each one it sets the `next` property. -// example action: { -// action: function to call, -// next: optional nested action object, would get overriden by this function, except for the last action, -// ... other attributes as required by the action function (for example ship or waypoing symbol...) -// } -export function chain(actions) { - for(let i=actions.length-1;i>0;--i) { - actions[i-1].next = actions[i]; - } - actions[0].action(actions[0]); -} - -// send takes a data object as argument -// example data: { +// send takes a request object as argument and an optional context ctx +// example request: { // endpoint: the url endpoint to call, // method: HTTP method for `fetch` call, defaults to 'GET', -// next: optional nested action object, as specified above with the chain function, // payload: optional json object that will be send along with the request, +// priority: optional priority value (defaults to 10, lower than 10 means the message will be sent faster) // } -export function send(data) { - if (!busy) { - send_this(data); - } else { - queue.enqueue(data, data.priority ? data.priority : 10); - } +export function send(request, ctx) { + return new Promise((resolve, reject) => { + let data = { + ctx: ctx, + reject: reject, + request: request, + resolve: resolve, + }; + if (!busy) { + busy = true; + send_this(data); + } else { + queue.enqueue(data, request.priority ? request.priority : 10); + } + }); } function send_next() { @@ -41,6 +37,7 @@ function send_next() { } } +// send_this take a data object as argument built in the send function above function send_this(data) { if (headers === undefined) { const token = getToken(); @@ -55,22 +52,20 @@ function send_this(data) { let options = { headers: headers, }; - if (data.method !== undefined) { - options['method'] = data.method; + if (data.request.method !== undefined) { + options['method'] = data.request.method; } - if (data.payload !== undefined) { - options['body'] = JSON.stringify(data.payload); + if (data.request.payload !== undefined) { + options['body'] = JSON.stringify(data.request.payload); } - busy = true; - fetch(`https://api.spacetraders.io/v2${data.endpoint}`, options) + fetch(`https://api.spacetraders.io/v2${data.request.endpoint}`, options) .then(response => response.json()) - .then(response => { - if (data.next !== undefined) { // if we have a next action, call it now - data.next.action(data.next, response); - } else { // otherwise use this debug action - console.log(JSON.stringify(response, null, 2)); - } - }) - .catch(err => console.error(err)); + .then(response => data.resolve(response)) + .catch(err => data.reject(err)); setTimeout(send_next, 500); } + +export function debugLog(ctx) { + console.log(`--- ${Date()} -----------------------------------------------------------------------------`); + console.log(JSON.stringify(ctx, null, 2)); +} diff --git a/lib/systems.js b/lib/systems.js new file mode 100644 index 0000000..ad354e4 --- /dev/null +++ b/lib/systems.js @@ -0,0 +1,36 @@ +import * as api from './api.js'; +import * as db from '../database/systems.js'; + +// Retrieves a list of waypoints that have a specific ctx.trait like a SHIPYARD or a MARKETPLACE in the system ctx.symbol +export async function trait(ctx) { + const s = await getSystem(ctx); + return s.filter(s => s.traits.some(t => t.symbol === ctx.trait)); +} + +// Retrieves a list of waypoints that have a specific ctx.type like ASTEROID_FIELD in the system ctx.symbol +export async function type(ctx, response) { + const s = await getSystem(ctx); + return s.filter(s => s.type === ctx.type); +} + +// Retrieves the system's information for ctx.symbol and cache it in the database +async function getSystem(ctx) { + let s = db.getSystem(ctx.symbol); + if (s === null) { + const response = await api.send({endpoint: `/systems/${ctx.symbol}/waypoints?limit=20&page=1`}); + if (response.error !== undefined) { + switch(response.error.code) { + case 404: + throw `Error retrieving waypoints for system ${ctx.symbol}: ${response.error.message}`; + default: // yet unhandled error + throw response; + } + } + if (response.meta !== undefined && response.meta.total > response.meta.limit) { + throw `Error retrieving waypoints for system ${ctx.symbol}: Pagination is not implemented yet!`; + } + s = response.data; + db.setSystem(ctx.symbol, s); + } + return s; +} diff --git a/main.js b/main.js index 6be7931..e19a059 100755 --- a/main.js +++ b/main.js @@ -1,6 +1,7 @@ import * as agent from './lib/agent.js'; import * as api from './lib/api.js'; import * as ships from './lib/ships.js'; +import * as systems from './lib/systems.js'; function usage() { console.log(`contracts\t\t\tList all of your contracts. @@ -72,11 +73,11 @@ default: case 'sell': ships.sell({ship: process.argv[3], good: process.argv[4], units: process.argv[5]}); break; - case 'shipyard': - api.send({endpoint: `/systems/${process.argv[3]}/waypoints/${process.argv[4]}/shipyard`}); + case 'asteroids': + api.debugLog(await systems.type({symbol: process.argv[3], type: 'ASTEROID_FIELD'})); break; - case 'waypoints': - api.send({endpoint: `/systems/${process.argv[3]}/waypoints?limit=20&page=1`}); + case 'shipyards': + api.debugLog(await systems.trait({symbol: process.argv[3], trait: 'SHIPYARD'})); break; default: usage(); -- cgit v1.2.3