How could this algorithm be optimized?

const ref = 5;

function getModulo(num) {
    let len = num.toString().length;
    let mod = 1
    while(len !== 1)  {
        mod *= 10;
        len -= 1
    }
    return mod
}

function recBigNumForNeg(num) {
    if(parseInt(num/mod) >= ref) {
        return '5' + num;
    }
}

function recBigNumForPos(num, mod) {
    if(num === 0) {
        return ref
    }
    if(parseInt(num/mod) < ref) {
        return '5' + num;
    }
    return '' + parseInt(num/mod) + recBigNumForPos(parseInt(num%mod), parseInt(mod/10))
}

function biggestNumber(num) {
    let mod = getModulo(num)
    if(num < 0) {
        return recBigNumForNeg(-num, mod)
    } else {        
        return recBigNumForPos(num, mod)
    }
}

The above code solves your problem, and it doesn't include any string related operations. But it does have couple of problems though.

1. The digits involved in the number should be only from 1 to 9, but it should not include 0. Since this is not mentioned in the question I took the privilege to not include 0.
2. recBigNumForNeg is the same method, but just a reversal in logic, so I put it as blank for your exercise.

This is just an alternative approach to the question, that's all.

Correctness:

If the input is -237, your function will return 2357 instead of -2357. This can be easily fixed by changing the return statement to

  return neg ? "-" + split.reverse().join('') : split.join('');

Performance:

Instead of converting the input to string, I would do MOD and DIV operations to split the input into left and right part and then insert 5 in between. I would do this for all possible positions for 5. Finally, I would pick the maximum value.

Example splitting 237:

1st iteration: left=237; right=0   -> num=2375
2nd iteration: left=23;  right=7   -> num=2357
2nd iteration: left=2;   right=37  -> num=2573
2nd iteration: left=0;   right=237 -> num=5237

The code:

function insertDigit(num) {
  var digit = num > 0 ? 5 : -5;
  var max = num * 10 + digit;
  var digitPos = 1;
  var left = num;
  var right = 0;
  
  while (left != 0) {
    right = num % digitPos;
    left = (num - right) * 10;

    max = Math.max(max, left + digitPos * digit + right)
    digitPos = digitPos * 10;
    }
  
  return max;
}

The only thing I could think is to break out of the loop earlier if split value is less than 5, to save some CPU cycles.

function insertDigit2(num) {
  let neg = num < 0;
  let split = neg 
    ? (num * -1).toString().split('').reverse()
    : num.toString().split('');
  
  let digit = 5;
  let i = 0;

  split.every(s => {
    if (s <= 5)
      return false;

    i++;
    return true
  })

  split.splice( i, 0, "5" );
  return neg ? split.reverse().join('') : split.join('');
}

console.log( insertDigit2(237) );