using a sync queue to throttle requests
xxxxxxxxxx
// Including the async module
const async = require('async');
// Creating an array for all elements execution
const tasks = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
// Initializing the queue
const queue = async.queue((task, executed) => {
console.log("Currently Busy Processing Task " + task);
setTimeout(()=>{
// Number of tasks remaining and to be processed
const tasksRemaining = queue.length();
executed(null, {task, tasksRemaining});
}, 1000);
}, 1); // concurrency value = 1
// Queue is idle initially as no elements are there...
console.log(`Queue Started ? ${queue.started}`)
// Adding each task from the tasks list
tasks.forEach((task)=>{
// Adding the task 5 to the head for priority execution
if(task == 5){
queue.unshift(task, (error, {task, tasksRemaining})=>{
if(error){
console.log(`An error occurred while processing task ${task}`);
}else {
console.log(`Finished processing task ${task}. ${tasksRemaining} tasks remaining`);
}
})
// Adding all the tasks at tail to be executed except task 5
} else {
queue.push(task, (error, {task, tasksRemaining})=>{
if(error){
console.log(`An error occurred while processing task ${task}`);
}else {
console.log(`Finished processing task ${task}. ${tasksRemaining}
tasks remaining`);
}
})
}
});
// Executes the callback when the queue is done processing all the tasks
queue.drain(() => {
console.log('All items are succesfully processed !');
})
// Checking if the queue is started after adding tasks
console.log(`Queue Started ? ${queue.started}`)
/*
console.log(queue.length());
// Executes when the queue is done processing all the items
queue.drain(() => {
console.log('Successfully processed all items');
})
// Pauses the execution of the queue
queue.pause()
// Resumes the execution of the queue
queue.resume()
// Forces the queue into idle mode
// and removes the drain callback
queue.kill()
// Returns whether the queue is idle or not
queue.idle()
Queue Started ? False
Queue Started ? True
Currently Busy Processing Task 5
Finished processing task 5. 9 tasks remaining
Currently Busy Processing Task 1
Finished processing task 1. 8 tasks remaining
Currently Busy Processing Task 2
Finished processing task 2. 7 tasks remaining
Currently Busy Processing Task 3
Finished processing task 3. 6 tasks remaining
Currently Busy Processing Task 4
Finished processing task 4. 5 tasks remaining
Currently Busy Processing Task 6
Finished processing task 6. 4 tasks remaining
Currently Busy Processing Task 7
Finished processing task 7. 3 tasks remaining
Currently Busy Processing Task 8
Finished processing task 8. 2 tasks remaining
Currently Busy Processing Task 9
Finished processing task 9. 1 tasks remaining
Currently Busy Processing Task 10
Finished processing task 10. 0 tasks remaining
All items are succesfully processed !
*/
xxxxxxxxxx
function createQueue(tasks, maxNumOfWorkers = 4) {
var numOfWorkers = 0;
var taskIndex = 0;
return new Promise(done => {
const handleResult = index => result => {
tasks[index] = result;
numOfWorkers--;
getNextTask();
};
const getNextTask = () => {
if (numOfWorkers < maxNumOfWorkers && taskIndex < tasks.length) {
tasks[taskIndex]().then(handleResult(taskIndex)).catch(handleResult(taskIndex));
taskIndex++;
numOfWorkers++;
getNextTask();
} else if (numOfWorkers === 0 && taskIndex === tasks.length) {
done(tasks);
}
};
getNextTask();
});
}