Why is the SFTP connector so much slower than the basic SFTP client

I’m using this sample class to test out streaming over SFTP. What I ultimately want to do is use a sink which is another SFTP server and stream from source(sftp server A) to sink(sftp server B) with my app in the middle. But for now, I’m testing throughput with Sink.ignore and the performance is much worse than just a plain SFTP client writing the file locally. Can someone tell me if this is a configuration problem or if I’m just “doing it wrong”

The timings are with a 10MB file pulling data from a VM in AWS to my laptop.

import akka.actor.ActorSystem
import akka.stream.alpakka.ftp.scaladsl.Sftp
import akka.stream.alpakka.ftp.{FtpCredentials, SftpSettings}
import akka.stream.scaladsl.Sink
import akka.stream.{ActorMaterializer, ActorMaterializerSettings, Supervision}
import com.workday.scala.logging.Slogger
import net.schmizz.sshj.transport.verification.PromiscuousVerifier
import net.schmizz.sshj.userauth.method.AuthPassword
import net.schmizz.sshj.userauth.password.{PasswordFinder, Resource}
import net.schmizz.sshj.{DefaultConfig, SSHClient}

import java.net.InetAddress
import scala.concurrent.Await
import scala.concurrent.duration.DurationInt

class StreamingSftpTransport {
 implicit val system: ActorSystem = ActorSystem("my-service")

 private val PORT = 22
 private val USER = "conor.griffin"
 private val CREDENTIALS = FtpCredentials.create(USER, "password!")
 private val BASE_PATH = s"/Users/$USER"
 private val FILE_NAME = "10mfile"
 private val CHUNK_SIZE = 131072

 // Set up the source system connection
 private val SOURCE_HOSTNAME = "suv1"

 private val sourceSettings = SftpSettings(host = InetAddress.getByName(SOURCE_HOSTNAME))
   .withCredentials(FtpCredentials.create("testsftp", "t3st123"))
   .withPort(PORT)
   .withStrictHostKeyChecking(false)

 private val sourceClient = new SSHClient(new DefaultConfig) {

 }
 private val configuredSourceClient = Sftp(sourceClient)


 // Set up the destination system connection

 private val DEST_HOSTNAME = "localhost"
 private val destSettings = SftpSettings(host = InetAddress.getByName(DEST_HOSTNAME))
   .withCredentials(CREDENTIALS)
   .withPort(PORT)
   .withStrictHostKeyChecking(false)

 private val destClient = new SSHClient(new DefaultConfig)
 private val configuredDestClient = Sftp(destClient)

 private val decider: Supervision.Decider = {
   case a => print(a.getMessage)
     Supervision.resume
 }

 implicit val materializer = ActorMaterializer(
   ActorMaterializerSettings(system).withSupervisionStrategy(decider))


 def doTransfer(): Unit = {
   println("Streaming")
   val source = configuredSourceClient.fromPath(s"/home/testsftp/$FILE_NAME", sourceSettings, CHUNK_SIZE)
   // val sink = configuredDestClient.toPath(s"$BASE_PATH/$FILE_NAME.out", destSettings)
   val runnable = source
     .runWith(Sink.ignore)

   println("Streaming: Starting")
   val start = System.currentTimeMillis()
   Await.result(runnable, 180 seconds)
   val end = System.currentTimeMillis()
   println(s"Streaming: ${end - start}")

 }

 def doSftpTransfer(): Unit = {
   println("SFTP")
   val ssh = new SSHClient(new DefaultConfig)
   ssh.addHostKeyVerifier(new PromiscuousVerifier)
   ssh.connect(SOURCE_HOSTNAME, 22)
   val passwordAuth: AuthPassword = new AuthPassword(new PasswordFinder() {
     def reqPassword(resource: Resource[_]): Array[Char] = "t3st123".toCharArray
     def shouldRetry(resource: Resource[_]) = false
   })
   ssh.auth("testsftp", passwordAuth)

   println("SFTP: Starting")
   val start = System.currentTimeMillis()
   ssh.newSFTPClient().get("/home/testsftp/10mfile", "/Users/conor.griffin/Downloads/10mfile.sftp")
   val end = System.currentTimeMillis()
   println(s"SFTP: ${end - start}")

 }

}

Output

[18-Dec-2020 19:41:00.994 UTC] INFO <Slf4jLogger> Slf4jLogger started 
[18-Dec-2020 19:41:02.672 UTC] INFO <BouncyCastleRandom> Generating random seed from SecureRandom. 
[18-Dec-2020 19:41:02.732 UTC] INFO <BouncyCastleRandom> Generating random seed from SecureRandom. 
Streaming
Streaming: Starting
[18-Dec-2020 19:41:03.060 UTC] INFO <TransportImpl> Client identity string: SSH-2.0-SSHJ_0.27.0 
[18-Dec-2020 19:41:03.434 UTC] INFO <TransportImpl> Server identity string: SSH-2.0-OpenSSH_7.4 
[18-Dec-2020 19:41:05.117 UTC] INFO <SessionChannel> Will request `sftp` subsystem 
[18-Dec-2020 19:41:57.819 UTC] INFO <TransportImpl> Disconnected - BY_APPLICATION 
Streaming: 55019
SFTP
[18-Dec-2020 19:41:57.826 UTC] INFO <BouncyCastleRandom> Generating random seed from SecureRandom. 
[18-Dec-2020 19:41:58.032 UTC] INFO <TransportImpl> Client identity string: SSH-2.0-SSHJ_0.27.0 
[18-Dec-2020 19:41:58.247 UTC] INFO <TransportImpl> Server identity string: SSH-2.0-OpenSSH_7.4 
SFTP: Starting
[18-Dec-2020 19:41:59.869 UTC] INFO <SessionChannel> Will request `sftp` subsystem 
SFTP: 12987

1 post - 1 participant

Read full topic