node-ftp icon indicating copy to clipboard operation
node-ftp copied to clipboard

After reaching 100% of downloading, script doesn't continue. Happens only for large files.

Open kunokdev opened this issue 9 years ago • 1 comments

I have a method to download file from FTP server and it works fine on smaller files, but when I use it to download file of ~5GB size of zip type, it downloads it, but after that it doesn't do anything. When it reaches 100% of downloading, then script doesn't continue. Should I wait if it's actually doing something in the background after download is complete? Is there filesize limit?

downloadFile: params => {
    return new Promise((resolve, reject) => {
      let ftpClient = new FTP()
      let total = params.state.fileSize
      let progress = 0
      ftpClient.on('ready', _ => {
        console.log(`Downloading ${params.targetedFile} ...`);
        ftpClient.get(params.targetedFile, (err, stream) => {
          if (err) reject(err)
          stream.on('data', buffer => {
            progress += buffer.length
            process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total})  \r`)
          })
          stream.once('close', _ => {
            ftpClient.end()
            console.log(`Saved downloaded file to ${params.localDir}`);
            resolve(params.localDir)
          })
          stream.pipe(fs.createWriteStream(params.localDir))
        })
      })
      ftpClient.connect(params.auth)
    })
  }

Basically, the callback for stream.on('close', ...) doesn't get executed when large file is downloaded. And it gets executed for smaller file of same type.

kunokdev avatar Apr 26 '17 13:04 kunokdev

If someone is looking for a workaround to this issue here modified code from the question:

/**
   * @name downloadFile
   * @desc downloads file from FTP server
   * @param  params, Object of params
   *   @prop auth: object, or null, authorization params
   *   @prop targetedFile: {String} filename e.g. data.txt
   *   @prop localDir: {String} filename on local disk
   *   @prop state: {Object} fileinfo object, {Int} .fileSize property is required
   * @return Promise, resolves given localDir
   */
  downloadFile: params => {
    return new Promise((resolve, reject) => {
      // validate param types
      if(typeof params.auth !== 'object'
      || typeof params.targetedFile !== 'string'
      || typeof params.localDir !== 'string'
      || typeof params.state !== 'object'
      || typeof params.state.fileSize !== 'number'
      ) throw new Error('You are either missing properties or passed wrong types')

      // initialize
      let ftpClient = new FTP()
      let total = params.state.fileSize
      let progress = 0

      //
      ftpClient.on('ready', _ => {
        console.log(`Downloading ${params.targetedFile} ...`)
        // get file
        ftpClient.get(params.targetedFile, (err, stream) => {
          if (err){
            ftpClient.end()
            return reject(err)
          }

          // upon data receive
          stream.on('data', buffer => {
            progress += buffer.length
            // if progress is complete
            if(progress === total){
              // start checking if local filesize matches server filesize
              let interval = setInterval(_ => {
                if(fs.statSync(params.localDir).size === total){
                  console.log(`Downloading file complete. Location: ${params.localDir}`);
                  clearInterval(interval)
                  ftpClient.end()
                  resolve(params.localDir)
                }
              })
            }
            // show current progress in percentages and bytes
            process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total})  \r`)
          })
          // pipe writestream to filesystem to write these bytes
          stream.pipe(fs.createWriteStream(params.localDir))
        })
      })
      ftpClient.connect(params.auth)
    })//promise
  }

kunokdev avatar May 02 '17 10:05 kunokdev