I have now just checked manually whether the cursor is at the border of scrolloff, i.e. if it would be scrolled by `<C-y>` for example or not, and if not, then I move it manually at each iteration with `j` or `k`:

    function SmoothScroll(up)
      if a:up
        let scrollaction= "\<C-y>"
        " now also check how far the cursor is away
        " from the border, including scrolloff
        let offdist = abs((line("w$") - &scrolloff) - line("."))
        if offdist > 0
          let cursormov="k"
        endif
      else
        let scrollaction="\<C-e>"
        let offdist = abs(line(".") - (line("w0") + &scrolloff))
        if offdist > 0
          let cursormov="j"
        endif
      endif
      let s=&scroll " total lines to scroll
      let x=0   " counts from 0 to s
      let T=10 " maximum sleep ms for smoothing
      let a=1 " smoothing amount
      let p=2 " must be at least 2
      let m=s/p " number of lines for which is being smoothed
      while x < s
        let time=T*((x-m)*(x-s+m)*(x+m/a)*(x-s-m/a))/(m*(s-m)*(-m/a)*(s+m/a))
        if time > 0
          exec "sleep ".time."m"
          redraw
        end
        let x+=1
        exec "normal " . scrollaction
        exec "normal " . cursormov
      endwhile
    endfunction
    nnoremap <C-U> :call SmoothScroll(1)<Enter>
    nnoremap <C-D> :call SmoothScroll(0)<Enter>
    inoremap <C-U> <Esc>:call SmoothScroll(1)<Enter>i
    inoremap <C-D> <Esc>:call SmoothScroll(0)<Enter>i


There may be a more efficient solution though. But this way the cursor also stays in place while smoothly scrolling.