adyxax/spacetraders
1
0
Fork 0
Code Issues Activity

[javascript] implemented paginated requests

Browse source
This commit is contained in:
Julien Dessaux 2023-09-01 23:51:51 +02:00
parent d78d9b1663
commit a50ea1be7c
Signed by: adyxax
GPG key ID: F92E51B86E07177E
1 changed files with 29 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
nodejs/lib/api.js
View file

@ -48,10 +48,29 @@ queue_processor();
// example request: {
// endpoint: the path part of the url to call,
// method: HTTP method for `fetch` call, defaults to 'GET',
// page: run a paginated request starting from this page until all the following pages are fetched
// payload: optional json object that will be send along with the request,
// priority: optional priority value (defaults to 10, lower than 10 means the message will be sent faster)
// }
export function send(request, ctx) {
export async function send(request, ctx) {
if (request.page === undefined) {
return await send_one(request, ctx);
}
let ret = [];
while (true) {
const response = await send_one(request, ctx);
if (response.meta === undefined) {
throw {"message": "paginated request did not return a meta block", "request": request, "response": response};
}
ret = ret.concat(response.data);
if (response.meta.limit * response.meta.page >= response.meta.total) {
return ret;
}
request.page++;
}
}
function send_one(request, ctx) {
return new Promise((resolve, reject) => {
let data = {
ctx: ctx,
@ -87,17 +106,21 @@ async function send_this(data) {
if (data.request.payload !== undefined) {
options['body'] = JSON.stringify(data.request.payload);
}
let pagination = "";
if (data.request.page !== undefined) {
pagination=`?limit=20&page=${data.request.page}`;
}
fs.writeFileSync('log', JSON.stringify({event: 'send', date: new Date(), data: data}) + '\n', {flag: 'a+'});
try {
let response = await fetch(`https://api.spacetraders.io/v2${data.request.endpoint}`, options);
let response = await fetch(`https://api.spacetraders.io/v2${data.request.endpoint}${pagination}`, options);
response = await response.json();
switch(response.error?.code) {
//case 401: // TODO 401 means a server reset happened
case 401: // TODO 401 means a server reset happened
throw response;
// TODO reject all promises in queue
// close database file
// rm database file
// reset database
// logrotate
// spawnSync
// spawnSync?
// break;
case 429: // 429 means rate limited, let's hold back as instructed
backoffSeconds = response.error.data.retryAfter;