Dv 5x sftp driver get size

datavault-broker/pom.xml

@@ -77,13 +77,7 @@
         <dependency>
             <groupId>org.mariadb.jdbc</groupId>
             <artifactId>mariadb-java-client</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-            <scope>runtime</scope>
+            <version>${maria.java.client.version}</version>
         </dependency>
 
         <dependency>

datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java

@@ -4,14 +4,18 @@
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.datavaultplatform.broker.app.DataVaultBrokerApp;
+import org.datavaultplatform.broker.config.MockServicesConfig;
 import org.datavaultplatform.broker.queue.Sender;
 import org.datavaultplatform.broker.services.FileStoreService;
 import org.datavaultplatform.broker.test.AddTestProperties;
-import org.datavaultplatform.broker.test.BaseDatabaseTest;
 import org.datavaultplatform.broker.test.TestClockConfig;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
+import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
@@ -32,10 +36,12 @@
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
 
 @SpringBootTest(classes = DataVaultBrokerApp.class)
-@Import(TestClockConfig.class)
+@Import({TestClockConfig.class, MockServicesConfig.class})
 @AddTestProperties
 @Slf4j
 @TestPropertySource(properties = {
+    "broker.services.enabled=false",
+    "broker.database.enabled=false",
     "broker.email.enabled=true",
     "broker.controllers.enabled=true",
     "broker.initialise.enabled=true",
@@ -44,7 +50,11 @@
     "management.endpoints.web.exposure.include=*",
     "management.health.rabbit.enabled=false"})
 @AutoConfigureMockMvc
-public class ActuatorTest extends BaseDatabaseTest {
+@EnableAutoConfiguration(exclude= {
+        DataSourceAutoConfiguration.class,
+        DataSourceTransactionManagerAutoConfiguration.class,
+        HibernateJpaAutoConfiguration.class })
+public class ActuatorTest {
 
   @Autowired
   MockMvc mvc;
@@ -55,7 +65,7 @@ public class ActuatorTest extends BaseDatabaseTest {
   @Autowired
   ObjectMapper mapper;
 
-  @MockBean
+  @Autowired
   FileStoreService mFileStoreService;
 
   @Test

datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemPrivatePublicKeyPairSizeIT.java

@@ -0,0 +1,103 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.security.interfaces.RSAPublicKey;
+import java.util.Map;
+import javax.crypto.SecretKey;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.bouncycastle.util.encoders.Base64;
+import org.datavaultplatform.broker.services.UserKeyPairService;
+import org.datavaultplatform.broker.services.UserKeyPairService.KeyPairInfo;
+import org.datavaultplatform.broker.services.UserKeyPairServiceJSchImpl;
+import org.datavaultplatform.common.PropNames;
+import org.datavaultplatform.common.crypto.Encryption;
+import org.datavaultplatform.common.crypto.SshRsaKeyUtils;
+import org.datavaultplatform.common.docker.DockerImage;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+
+@Slf4j
+public abstract class BaseSFTPFileSystemPrivatePublicKeyPairSizeIT extends BaseSFTPFileSystemSizeIT {
+
+  static final String TEST_PASSPHRASE = "tenet";
+  static final String ENV_PUBLIC_KEY = "PUBLIC_KEY";
+  static final String KEY_STORE_PASSWORD = "keyStorePassword";
+  static final String SSH_KEY_NAME = "sshKeyName";
+
+  static File keyStoreTempDir;
+  static KeyPairInfo keyPairInfo;
+
+
+
+  static GenericContainer<?> initialiseContainer(String tcName) {
+
+    try {
+      keyStoreTempDir = Files.createTempDirectory("tmpKeyStoreDir").toFile();
+    }catch(IOException ex){
+      throw new RuntimeException(ex);
+    }
+
+    keyPairInfo = generateKeyPair();
+
+    return new GenericContainer<>(DockerImage.OPEN_SSH_8pt6_IMAGE_NAME)
+        .withEnv("TC_NAME", tcName)
+        .withEnv(ENV_USER_NAME, TEST_USER)
+        .withEnv(ENV_PUBLIC_KEY,
+            keyPairInfo.getPublicKey()) //this causes the public key to be added to /config/.ssh/authorized_keys
+        .withExposedPorts(SFTP_SERVER_PORT)
+        //.withFileSystemBind("/Users/davidhay/SPARSE_FILES/files","/tmp/files", BindMode.READ_ONLY)
+        .waitingFor(Wait.forListeningPort());
+  }
+
+
+  @SneakyThrows
+  @Override
+  public void addAuthenticationProps(Map<String, String> props) {
+    byte[] iv = Encryption.generateIV();
+    byte[] encrypted = Encryption.encryptSecret(keyPairInfo.getPrivateKey(), null, iv);
+
+    props.put(PropNames.PASSPHRASE, TEST_PASSPHRASE);
+    props.put(PropNames.IV, Base64.toBase64String(iv));
+    props.put(PropNames.PRIVATE_KEY, Base64.toBase64String(encrypted));
+
+    RSAPublicKey publicKey = SshRsaKeyUtils.readPublicKey(
+        keyPairInfo.getPublicKey());
+    if(getLog().isTraceEnabled()) {
+      getLog().trace("ORIG PUBLIC KEY MODULUS [{}]", publicKey.getModulus().toString(16));
+    }
+  }
+
+  @SneakyThrows
+  private static KeyPairInfo generateKeyPair() {
+
+    Encryption.addBouncyCastleSecurityProvider();
+    String keyStorePath = keyStoreTempDir.toPath().resolve("test.ks").toString();
+    log.info("TEMP KEY IS AT [{}]", keyStorePath);
+
+    Encryption enc = new Encryption();
+    enc.setVaultEnable(false);
+    enc.setVaultPrivateKeyEncryptionKeyName(SSH_KEY_NAME);
+
+    enc.setKeystoreEnable(true);
+    enc.setKeystorePath(keyStorePath);
+    enc.setKeystorePassword(KEY_STORE_PASSWORD);
+
+    SecretKey keyForKeyStore = Encryption.generateSecretKey();
+
+    assertFalse(new File(keyStorePath).exists());
+
+    // Encryption class uses 'vaultPrivateKeyEncryptionKeyName' property as the default key name for JavaKeyStore
+    Encryption.saveSecretKeyToKeyStore(Encryption.getVaultPrivateKeyEncryptionKeyName(),
+        keyForKeyStore);
+
+    assertTrue(new File(keyStorePath).exists());
+    UserKeyPairService userKeyPairService = new UserKeyPairServiceJSchImpl(TEST_PASSPHRASE);
+    return userKeyPairService.generateNewKeyPair();
+  }
+}

datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemSizeIT.java

@@ -0,0 +1,58 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.SneakyThrows;
+import org.datavaultplatform.common.PropNames;
+import org.datavaultplatform.common.storage.SFTPFileSystemDriver;
+import org.slf4j.Logger;
+import org.testcontainers.containers.GenericContainer;
+
+public abstract class BaseSFTPFileSystemSizeIT {
+
+  static final int SFTP_SERVER_PORT = 2222;
+
+  static final String ENV_USER_NAME = "USER_NAME";
+  static final String TEST_USER = "testuser";
+
+  static final String SFTP_ROOT_DIR = "/config";
+  static final Clock TEST_CLOCK = Clock.fixed(Instant.parse("2022-03-26T09:44:33.22Z"),
+      ZoneId.of("Europe/London"));
+  static final String TEMP_PREFIX = "dvSftpTempDir";
+  static final long EXPECTED_SPACE_AVAILABLE_ON_SFTP_SERVER = 100_000;
+  private static final int FREE_SPACE_FACTOR = 10;
+
+  public abstract GenericContainer<?> getContainer();
+
+  public abstract SFTPFileSystemDriver getSftpDriver();
+
+  public abstract void addAuthenticationProps(Map<String,String> props);
+
+  public final int getSftpServerPort() {
+    return getContainer().getMappedPort(SFTP_SERVER_PORT);
+  }
+
+  public final String getSftpServerHost() {
+    return getContainer().getHost();
+  }
+
+  @SneakyThrows
+  protected Map<String, String> getStoreProperties() {
+    HashMap<String, String> props = new HashMap<>();
+
+    //standard sftp properties
+    props.put(PropNames.USERNAME, TEST_USER);
+    props.put(PropNames.ROOT_PATH, SFTP_ROOT_DIR); //this is the directory ON THE SFTP SERVER - for ALL OpenSSH containers, it's config
+    props.put(PropNames.HOST, getSftpServerHost());
+    props.put(PropNames.PORT, String.valueOf(getSftpServerPort()));
+
+    addAuthenticationProps(props);
+
+    return props;
+  }
+
+  abstract Logger getLog();
+}

datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/CombinedSizeSFTPJSchIT.java

@@ -0,0 +1,122 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Map;
+import java.util.stream.Stream;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.SFTPFileSystemDriver;
+import org.datavaultplatform.common.storage.impl.SFTPFileSystemJSch;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.testcontainers.containers.Container.ExecResult;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.MountableFile;
+
+@Testcontainers(disabledWithoutDocker = true)
+@Slf4j
+public class CombinedSizeSFTPJSchIT extends BaseSFTPFileSystemPrivatePublicKeyPairSizeIT {
+
+  @Container
+  static final GenericContainer<?> container = initialiseContainer("SftpPrivatePublicJSchDIT");
+  private static final int BYTES_PER_FILE = 4;
+
+  @BeforeAll
+  @SneakyThrows
+  static void setupContainer() {
+    log.info("Copying script createFileTree.sh to /tmp in SFTP Container");
+    container.copyFileToContainer(MountableFile.forClasspathResource("sftpsize/createFileTree.sh"),
+        "/tmp/createFileTree.sh");
+    log.info("Creating directory /config/files in SFTP Container");
+    container.execInContainer("mkdir", "-p", "/config/files");
+  }
+
+  @BeforeEach
+  @SneakyThrows
+  void cleanupAnyTestFilesInContainer() {
+    container.execInContainer("/bin/bash", "-c", "cd /config/files; rm -rf *");
+  }
+
+  static Stream<Arguments> getTestCaseParams() {
+    return Stream.of(
+        TestCaseParam.SMALL
+        //, TestCaseParam.SMALLISH
+        //, TestCaseParam.MEDIUM
+        //, TestCaseParam.LARGE
+    ).map(param -> Arguments.of(param, String.format("%s files[%s] depth[%s]",param,param.fileCount, param.depth)));
+  }
+
+  @ParameterizedTest(name="{index}-{1}")
+  @MethodSource("getTestCaseParams")
+  @SneakyThrows
+  void testTotalSizeOfFilesViaSftp(TestCaseParam testCaseParam, String testDescription) {
+
+    log.info("Creating files beneath /config/files in SFTP Container");
+    container.execInContainer("/tmp/createFileTree.sh", "/config/files", ""+ testCaseParam.depth);
+    log.info("Created files beneath /config/files in SFTP Container");
+
+    // count the number of files in the beneath /config/files in the SFTP Container
+    ExecResult result = container.execInContainer(
+        "/bin/bash","-c","cd /config/files; find . -type f | wc -l");
+
+    log.info("output message [{}]", result.getStdout());
+    log.info("error message [{}]", result.getStderr());
+    log.info("exit code [{}]", result.getExitCode());
+
+    assertEquals(0, result.getExitCode());
+    assertEquals(testCaseParam.fileCount, Integer.parseInt(result.getStdout().trim()));
+
+    // check that the getSize method of SFTPDriver returns the expected size
+    log.info("BEFORE GET SIZE");
+
+    // get the total size files beneath /config/files
+    long size = getSftpDriver().getSize("files");
+    log.info("AFTER GET SIZE");
+    assertEquals(testCaseParam.fileCount * BYTES_PER_FILE, size);
+  }
+
+  /**
+   * @See DirectrorySizeTest
+   */
+  enum TestCaseParam {
+
+    SMALL(4, 62),
+    SMALLISH(10, 4094),
+    MEDIUM(13, 32766),
+    LARGE(17, 524286);
+
+    public final int depth;
+    public final long fileCount;
+
+    TestCaseParam(int depth, long fileCount) {
+      this.depth = depth;
+      this.fileCount = fileCount;
+    }
+
+
+  }
+
+  @Override
+  public SFTPFileSystemDriver getSftpDriver() {
+    Map<String, String> props = getStoreProperties();
+    return new SFTPFileSystemJSch("sftp-jsch", props, TEST_CLOCK);
+  }
+
+  @Override
+  public GenericContainer<?> getContainer() {
+    return container;
+  }
+
+  @Override
+  Logger getLog() {
+    return log;
+  }
+}