I'm really lost on this, I tried a lot of things like changing up the timeouts, using different libs etc, I hope someone can see why this happens with the actual code thats giving me trouble. I'll try to be concise.

Quick text explanation:

A worker picks up a job from a queue, uses a module, and transfers to it all the required info. The module then sends Promise.all for all the requests to another microservice with axios. This microservice then makes another https request to a service I don't control, which very often takes a long amount of time to answer (1-10 minutes). The service should return a document, which the microservice will then return to the module used by the worker.

It seems while waiting for the slow service to answer the microservice the event loop can't answer health checks made by kubernetes.

Code:

Worker's module:

const axios = require('axios');
const { getVariableValue } = require("redacted");
const logger = require('redacted');
const https = require('https')

module.exports = webApiOptiDocService = {

    async getDocuments(docIDs, transactionId, timeStamp) {

        try {

            var documents = [];

            await Promise.all(docIDs.map(async (docID) => {

                var document = await axios.post(getVariableValue("GET_DOCUMENT_SERVICE_URL"), docID, {
                    httpsAgent: new https.Agent({
                        rejectUnauthorized: false,
                        keepAlive: true
                    }),
                    auth: {
                        username: getVariableValue("WEBAPI_OPTIDOCS_SERVICE_USERNAME"),
                        password: getVariableValue("WEBAPI_OPTIDOCS_SERVICE_PASSWORD")
                    },
                    headers: {
                        "x-global-transaction-id": transactionId,
                        "timeStamp": timeStamp
                    }

                });

                documents.push(document.data.content);

            }));

            return documents

        }
        catch (err) {

            const responseData = err.response ? err.response.data : err.message

            throw Error(responseData)

        }

    }

}

This is the microservice that then gets these requests:

API:

const express = require('express');
const router = express.Router();
const logger = require('redacted');
const getDocuemntService = require('./getDocumentService')
var clone = require('clone')

module.exports = router.post('/getDocument', async (req, res, next) => {

    try {

        var transactionId = req.headers["x-global-transaction-id"]

        var timeStamp = req.headers["timestamp"]

        var document = await getDocuemntService(req.body, transactionId, timeStamp);

        var cloneDocument = clone(document)

        res.status(200);

        res.json({
            statusDesc: "Success",
            status: true,
            content: cloneDocument
        });

    }
    catch (err) {

         res.status(500).send("stack: " + err.stack + "err: " + err.message + " fromgetdocument")

    }

});

This is the getDocument.js module it then uses:

const axios = require('axios');
const { getVariableValue } = require("redacted");
const logger = require('redacted');
const https = require('https')
const fileType = require('file-type')
var request = require('request-promise-native')

module.exports = async (docID, transactionId, timeStamp) => {

    try {



        var startTime = Date.now();



        const documentBeforeParse = await request.get(getVariableValue("WEBAPI_OPTIDOCS_SERVICE_URL_GET") + docID.docID, {

            strictSSL: false,
            headers: {
                'x-global-transaction-id': transactionId
            },
            timeout: Infinity
        }).auth(getVariableValue("WEBAPI_OPTIDOCS_SERVICE_USERNAME"), getVariableValue("WEBAPI_OPTIDOCS_SERVICE_PASSWORD"))
        const parsedDocument = JSON.parse(documentBeforeParse)

        var document = {
            data: {
                mimeType: parsedDocument.mimeType,
                fileName: "",
                content: parsedDocument.content
            }
        }

        // const document = await axios.get(getVariableValue("WEBAPI_OPTIDOCS_SERVICE_URL_GET") + docID.docID, {
        //     maxContentLength: 524288000, //500 MB in Bytes
        //     maxBodyLength: 524288000, //500 MB in Bytes
        //     method: 'get',
        //     httpsAgent: new https.Agent({
        //         rejectUnauthorized: false
        //     }),
        //     auth: {
        //         username: getVariableValue("WEBAPI_OPTIDOCS_SERVICE_USERNAME"),
        //         password: getVariableValue("WEBAPI_OPTIDOCS_SERVICE_PASSWORD")
        //     },
        //     headers: {
        //         'x-global-transaction-id': transactionId
        //     }
        // });

        if (document.data.mimeType == "") {

            if (recoverMimeType(document.data.content)) {

                document.data.mimeType = recoverMimeType(document.data.content).ext

            }
            else {

                throw Error("Missing mime type can not be recovered.")

            }

        }

        var fixedMimeType = fixMimeType(document.data.mimeType);

        document.data.fileName = docID.docPartsPrefixName + fixedMimeType

        var returnDocument = {
            fileName: document.data.fileName,
            content: document.data.content,
            mimeType: document.data.mimeType
        }

        return returnDocument;

    }
    catch (error) {

        throw Error(error);

    }

}

function fixMimeType(mimeType) {
    return "." + mimeType
}

function recoverMimeType(content) {

    return fileType.fromBuffer(Buffer.from(content[0], "base64"))

}

I removed all loggers, but kept in the commented out axios request which I tried replacing for Request to test if that had anything to do with it.

Basically the microservice gets 50-150 requests through the Promise.all, and after fetching quite a lot, will die out eventually on the really slow answers because it won't answer health checks.