diff --git a/datavault-broker/pom.xml b/datavault-broker/pom.xml
index a677302c3..b94345fa9 100644
--- a/datavault-broker/pom.xml
+++ b/datavault-broker/pom.xml
@@ -77,13 +77,7 @@
org.mariadb.jdbc
mariadb-java-client
- test
-
-
-
- mysql
- mysql-connector-java
- runtime
+ ${maria.java.client.version}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java
index 9243a94c9..f07977b5c 100644
--- a/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java
+++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java
@@ -4,14 +4,18 @@
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.datavaultplatform.broker.app.DataVaultBrokerApp;
+import org.datavaultplatform.broker.config.MockServicesConfig;
import org.datavaultplatform.broker.queue.Sender;
import org.datavaultplatform.broker.services.FileStoreService;
import org.datavaultplatform.broker.test.AddTestProperties;
-import org.datavaultplatform.broker.test.BaseDatabaseTest;
import org.datavaultplatform.broker.test.TestClockConfig;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
+import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
@@ -32,10 +36,12 @@
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
@SpringBootTest(classes = DataVaultBrokerApp.class)
-@Import(TestClockConfig.class)
+@Import({TestClockConfig.class, MockServicesConfig.class})
@AddTestProperties
@Slf4j
@TestPropertySource(properties = {
+ "broker.services.enabled=false",
+ "broker.database.enabled=false",
"broker.email.enabled=true",
"broker.controllers.enabled=true",
"broker.initialise.enabled=true",
@@ -44,7 +50,11 @@
"management.endpoints.web.exposure.include=*",
"management.health.rabbit.enabled=false"})
@AutoConfigureMockMvc
-public class ActuatorTest extends BaseDatabaseTest {
+@EnableAutoConfiguration(exclude= {
+ DataSourceAutoConfiguration.class,
+ DataSourceTransactionManagerAutoConfiguration.class,
+ HibernateJpaAutoConfiguration.class })
+public class ActuatorTest {
@Autowired
MockMvc mvc;
@@ -55,7 +65,7 @@ public class ActuatorTest extends BaseDatabaseTest {
@Autowired
ObjectMapper mapper;
- @MockBean
+ @Autowired
FileStoreService mFileStoreService;
@Test
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemPrivatePublicKeyPairSizeIT.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemPrivatePublicKeyPairSizeIT.java
new file mode 100644
index 000000000..1d0577535
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemPrivatePublicKeyPairSizeIT.java
@@ -0,0 +1,103 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.security.interfaces.RSAPublicKey;
+import java.util.Map;
+import javax.crypto.SecretKey;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.bouncycastle.util.encoders.Base64;
+import org.datavaultplatform.broker.services.UserKeyPairService;
+import org.datavaultplatform.broker.services.UserKeyPairService.KeyPairInfo;
+import org.datavaultplatform.broker.services.UserKeyPairServiceJSchImpl;
+import org.datavaultplatform.common.PropNames;
+import org.datavaultplatform.common.crypto.Encryption;
+import org.datavaultplatform.common.crypto.SshRsaKeyUtils;
+import org.datavaultplatform.common.docker.DockerImage;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+
+@Slf4j
+public abstract class BaseSFTPFileSystemPrivatePublicKeyPairSizeIT extends BaseSFTPFileSystemSizeIT {
+
+ static final String TEST_PASSPHRASE = "tenet";
+ static final String ENV_PUBLIC_KEY = "PUBLIC_KEY";
+ static final String KEY_STORE_PASSWORD = "keyStorePassword";
+ static final String SSH_KEY_NAME = "sshKeyName";
+
+ static File keyStoreTempDir;
+ static KeyPairInfo keyPairInfo;
+
+
+
+ static GenericContainer initialiseContainer(String tcName) {
+
+ try {
+ keyStoreTempDir = Files.createTempDirectory("tmpKeyStoreDir").toFile();
+ }catch(IOException ex){
+ throw new RuntimeException(ex);
+ }
+
+ keyPairInfo = generateKeyPair();
+
+ return new GenericContainer<>(DockerImage.OPEN_SSH_8pt6_IMAGE_NAME)
+ .withEnv("TC_NAME", tcName)
+ .withEnv(ENV_USER_NAME, TEST_USER)
+ .withEnv(ENV_PUBLIC_KEY,
+ keyPairInfo.getPublicKey()) //this causes the public key to be added to /config/.ssh/authorized_keys
+ .withExposedPorts(SFTP_SERVER_PORT)
+ //.withFileSystemBind("/Users/davidhay/SPARSE_FILES/files","/tmp/files", BindMode.READ_ONLY)
+ .waitingFor(Wait.forListeningPort());
+ }
+
+
+ @SneakyThrows
+ @Override
+ public void addAuthenticationProps(Map props) {
+ byte[] iv = Encryption.generateIV();
+ byte[] encrypted = Encryption.encryptSecret(keyPairInfo.getPrivateKey(), null, iv);
+
+ props.put(PropNames.PASSPHRASE, TEST_PASSPHRASE);
+ props.put(PropNames.IV, Base64.toBase64String(iv));
+ props.put(PropNames.PRIVATE_KEY, Base64.toBase64String(encrypted));
+
+ RSAPublicKey publicKey = SshRsaKeyUtils.readPublicKey(
+ keyPairInfo.getPublicKey());
+ if(getLog().isTraceEnabled()) {
+ getLog().trace("ORIG PUBLIC KEY MODULUS [{}]", publicKey.getModulus().toString(16));
+ }
+ }
+
+ @SneakyThrows
+ private static KeyPairInfo generateKeyPair() {
+
+ Encryption.addBouncyCastleSecurityProvider();
+ String keyStorePath = keyStoreTempDir.toPath().resolve("test.ks").toString();
+ log.info("TEMP KEY IS AT [{}]", keyStorePath);
+
+ Encryption enc = new Encryption();
+ enc.setVaultEnable(false);
+ enc.setVaultPrivateKeyEncryptionKeyName(SSH_KEY_NAME);
+
+ enc.setKeystoreEnable(true);
+ enc.setKeystorePath(keyStorePath);
+ enc.setKeystorePassword(KEY_STORE_PASSWORD);
+
+ SecretKey keyForKeyStore = Encryption.generateSecretKey();
+
+ assertFalse(new File(keyStorePath).exists());
+
+ // Encryption class uses 'vaultPrivateKeyEncryptionKeyName' property as the default key name for JavaKeyStore
+ Encryption.saveSecretKeyToKeyStore(Encryption.getVaultPrivateKeyEncryptionKeyName(),
+ keyForKeyStore);
+
+ assertTrue(new File(keyStorePath).exists());
+ UserKeyPairService userKeyPairService = new UserKeyPairServiceJSchImpl(TEST_PASSPHRASE);
+ return userKeyPairService.generateNewKeyPair();
+ }
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemSizeIT.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemSizeIT.java
new file mode 100644
index 000000000..a2805d4e9
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/BaseSFTPFileSystemSizeIT.java
@@ -0,0 +1,58 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.SneakyThrows;
+import org.datavaultplatform.common.PropNames;
+import org.datavaultplatform.common.storage.SFTPFileSystemDriver;
+import org.slf4j.Logger;
+import org.testcontainers.containers.GenericContainer;
+
+public abstract class BaseSFTPFileSystemSizeIT {
+
+ static final int SFTP_SERVER_PORT = 2222;
+
+ static final String ENV_USER_NAME = "USER_NAME";
+ static final String TEST_USER = "testuser";
+
+ static final String SFTP_ROOT_DIR = "/config";
+ static final Clock TEST_CLOCK = Clock.fixed(Instant.parse("2022-03-26T09:44:33.22Z"),
+ ZoneId.of("Europe/London"));
+ static final String TEMP_PREFIX = "dvSftpTempDir";
+ static final long EXPECTED_SPACE_AVAILABLE_ON_SFTP_SERVER = 100_000;
+ private static final int FREE_SPACE_FACTOR = 10;
+
+ public abstract GenericContainer getContainer();
+
+ public abstract SFTPFileSystemDriver getSftpDriver();
+
+ public abstract void addAuthenticationProps(Map props);
+
+ public final int getSftpServerPort() {
+ return getContainer().getMappedPort(SFTP_SERVER_PORT);
+ }
+
+ public final String getSftpServerHost() {
+ return getContainer().getHost();
+ }
+
+ @SneakyThrows
+ protected Map getStoreProperties() {
+ HashMap props = new HashMap<>();
+
+ //standard sftp properties
+ props.put(PropNames.USERNAME, TEST_USER);
+ props.put(PropNames.ROOT_PATH, SFTP_ROOT_DIR); //this is the directory ON THE SFTP SERVER - for ALL OpenSSH containers, it's config
+ props.put(PropNames.HOST, getSftpServerHost());
+ props.put(PropNames.PORT, String.valueOf(getSftpServerPort()));
+
+ addAuthenticationProps(props);
+
+ return props;
+ }
+
+ abstract Logger getLog();
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/CombinedSizeSFTPJSchIT.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/CombinedSizeSFTPJSchIT.java
new file mode 100644
index 000000000..54623ee09
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/CombinedSizeSFTPJSchIT.java
@@ -0,0 +1,122 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Map;
+import java.util.stream.Stream;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.SFTPFileSystemDriver;
+import org.datavaultplatform.common.storage.impl.SFTPFileSystemJSch;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.testcontainers.containers.Container.ExecResult;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.MountableFile;
+
+@Testcontainers(disabledWithoutDocker = true)
+@Slf4j
+public class CombinedSizeSFTPJSchIT extends BaseSFTPFileSystemPrivatePublicKeyPairSizeIT {
+
+ @Container
+ static final GenericContainer container = initialiseContainer("SftpPrivatePublicJSchDIT");
+ private static final int BYTES_PER_FILE = 4;
+
+ @BeforeAll
+ @SneakyThrows
+ static void setupContainer() {
+ log.info("Copying script createFileTree.sh to /tmp in SFTP Container");
+ container.copyFileToContainer(MountableFile.forClasspathResource("sftpsize/createFileTree.sh"),
+ "/tmp/createFileTree.sh");
+ log.info("Creating directory /config/files in SFTP Container");
+ container.execInContainer("mkdir", "-p", "/config/files");
+ }
+
+ @BeforeEach
+ @SneakyThrows
+ void cleanupAnyTestFilesInContainer() {
+ container.execInContainer("/bin/bash", "-c", "cd /config/files; rm -rf *");
+ }
+
+ static Stream getTestCaseParams() {
+ return Stream.of(
+ TestCaseParam.SMALL
+ //, TestCaseParam.SMALLISH
+ //, TestCaseParam.MEDIUM
+ //, TestCaseParam.LARGE
+ ).map(param -> Arguments.of(param, String.format("%s files[%s] depth[%s]",param,param.fileCount, param.depth)));
+ }
+
+ @ParameterizedTest(name="{index}-{1}")
+ @MethodSource("getTestCaseParams")
+ @SneakyThrows
+ void testTotalSizeOfFilesViaSftp(TestCaseParam testCaseParam, String testDescription) {
+
+ log.info("Creating files beneath /config/files in SFTP Container");
+ container.execInContainer("/tmp/createFileTree.sh", "/config/files", ""+ testCaseParam.depth);
+ log.info("Created files beneath /config/files in SFTP Container");
+
+ // count the number of files in the beneath /config/files in the SFTP Container
+ ExecResult result = container.execInContainer(
+ "/bin/bash","-c","cd /config/files; find . -type f | wc -l");
+
+ log.info("output message [{}]", result.getStdout());
+ log.info("error message [{}]", result.getStderr());
+ log.info("exit code [{}]", result.getExitCode());
+
+ assertEquals(0, result.getExitCode());
+ assertEquals(testCaseParam.fileCount, Integer.parseInt(result.getStdout().trim()));
+
+ // check that the getSize method of SFTPDriver returns the expected size
+ log.info("BEFORE GET SIZE");
+
+ // get the total size files beneath /config/files
+ long size = getSftpDriver().getSize("files");
+ log.info("AFTER GET SIZE");
+ assertEquals(testCaseParam.fileCount * BYTES_PER_FILE, size);
+ }
+
+ /**
+ * @See DirectrorySizeTest
+ */
+ enum TestCaseParam {
+
+ SMALL(4, 62),
+ SMALLISH(10, 4094),
+ MEDIUM(13, 32766),
+ LARGE(17, 524286);
+
+ public final int depth;
+ public final long fileCount;
+
+ TestCaseParam(int depth, long fileCount) {
+ this.depth = depth;
+ this.fileCount = fileCount;
+ }
+
+
+ }
+
+ @Override
+ public SFTPFileSystemDriver getSftpDriver() {
+ Map props = getStoreProperties();
+ return new SFTPFileSystemJSch("sftp-jsch", props, TEST_CLOCK);
+ }
+
+ @Override
+ public GenericContainer getContainer() {
+ return container;
+ }
+
+ @Override
+ Logger getLog() {
+ return log;
+ }
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/BaseNestedFilesTest.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/BaseNestedFilesTest.java
new file mode 100644
index 000000000..5f4685e57
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/BaseNestedFilesTest.java
@@ -0,0 +1,65 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.path;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.PrintWriter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.io.TempDir;
+
+
+@Slf4j
+public abstract class BaseNestedFilesTest {
+
+ long fileCounter;
+
+ @TempDir
+ File baseDir;
+
+ @BeforeEach
+ void setup() {
+ fileCounter = 0;
+ assertTrue(baseDir.exists() && baseDir.isDirectory());
+ log.info("BASE DIR [{}]", baseDir);
+ }
+
+ void createFiles(File base, int depth) {
+ createFilesInternal(base, depth);
+ log.info("Created [{}] files", this.fileCounter);
+ }
+
+ private void createFilesInternal(File base, int depth) {
+ if (depth > 0) {
+ File left = new File(base, "left_" + depth);
+ left.mkdirs();
+ File right = new File(base, "right_" + depth);
+ right.mkdirs();
+ createFilesInternal(left, depth - 1);
+ createFilesInternal(right, depth - 1);
+ }
+ File aaa = new File(base, "aaa.txt");
+ write(aaa, "aaa");
+ File bbb = new File(base, "bbb.txt");
+ write(bbb, "bbb");
+ }
+
+
+ @SneakyThrows
+ void write(File file, String contents) {
+ try (PrintWriter pw = new PrintWriter(file)) {
+ pw.write(contents);
+ }
+ fileCounter++;
+ }
+
+ public static final long getExpectedFileCountAtLevel(int level) {
+ long total = 0;
+ for (int depth = 0; depth <= level; depth++) {
+ total += Math.pow(2, depth + 1);
+ }
+ log.info("expected file count for depth[{}] is [{}]", level, total);
+ return total;
+ }
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/CountFilesInDirectoryTest.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/CountFilesInDirectoryTest.java
new file mode 100644
index 000000000..3c01125e5
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/CountFilesInDirectoryTest.java
@@ -0,0 +1,79 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+/*
+This tests that the number of files created by createFilesInternal
+is the same calculated by getExpectedFileCountAtLevel.
+These (level, number of files) pairs are important for SftpDirectorySizeTest.
+ */
+@Slf4j
+public class CountFilesInDirectoryTest extends BaseNestedFilesTest {
+
+ @Test
+ void testLevel0() {
+ createFiles(this.baseDir, 0);
+ assertEquals(getExpectedFileCountAtLevel(0), this.fileCounter);
+ }
+
+ @Test
+ void testLevel1() {
+ createFiles(this.baseDir, 1);
+ assertEquals(getExpectedFileCountAtLevel(1), this.fileCounter);
+ }
+
+ @Test
+ void testLevel2() {
+ createFiles(this.baseDir, 2);
+ assertEquals(getExpectedFileCountAtLevel(2), this.fileCounter);
+ }
+
+ @Test
+ void testLevel3() {
+ createFiles(this.baseDir, 3);
+ assertEquals(getExpectedFileCountAtLevel(3), this.fileCounter);
+ }
+
+ @Test
+ void testLevel4() {
+ createFiles(this.baseDir, 4);
+ assertEquals(getExpectedFileCountAtLevel(4), this.fileCounter);
+ }
+
+ @Test
+ void testNumberOfFilesAtLevel10() {
+ assertEquals(4_094, getExpectedFileCountAtLevel(10));
+ }
+
+ @Test
+ void testNumberOfFilesAtLevel11() {
+ assertEquals(8_190, getExpectedFileCountAtLevel(11));
+ }
+
+ @Test
+ void testNumberOfFilesAtLevel12() {
+ assertEquals(16_382, getExpectedFileCountAtLevel(12));
+ }
+
+ @Test
+ void testNumberOfFilesAtLevel16() {
+ assertEquals(262_142, getExpectedFileCountAtLevel(16));
+ }
+
+ @Test
+ @Disabled
+ @SneakyThrows
+ void testLevels() {
+ for (int level = 0; level < 30; level++) {
+ long expected = getExpectedFileCountAtLevel(level);
+ log.info("level[{}] files[{}]", level, expected);
+ }
+ }
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/DirEntryPath.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/DirEntryPath.java
new file mode 100644
index 000000000..fa9a91380
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/DirEntryPath.java
@@ -0,0 +1,41 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.path;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.datavaultplatform.common.storage.impl.ssh.stack.Item;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+
+public class DirEntryPath implements Item {
+
+ private final File dirEntry;
+
+ public DirEntryPath(File dirEntry) {
+ this.dirEntry = dirEntry;
+ }
+
+ @Override
+ public void process(Stack> stack, ItemContext ctx) {
+ if (dirEntry.isFile()) {
+ ctx.increment(dirEntry.length());
+ ctx.incrementCount();
+ } else {
+ ctx.setContext(dirEntry.toPath());
+ Map> groupedItems = Stream.of(dirEntry.listFiles())
+ .collect(Collectors.partitioningBy(File::isDirectory));
+
+ //push directory 'pop'
+ stack.push(EndDirItemPath.getInstance());
+
+ //push all directories
+ groupedItems.get(true).forEach(dir -> stack.push(new DirEntryPath(dir)));
+
+ //push all 'non directories'
+ groupedItems.get(false).forEach(nonDir -> stack.push(new DirEntryPath(nonDir)));
+ }
+ }
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/EndDirItemPath.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/EndDirItemPath.java
new file mode 100644
index 000000000..aae2919db
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/EndDirItemPath.java
@@ -0,0 +1,27 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.path;
+
+import java.nio.file.Path;
+import java.util.Stack;
+import org.datavaultplatform.common.storage.impl.ssh.stack.Item;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+
+public class EndDirItemPath implements Item {
+
+ private static final EndDirItemPath INSTANCE = new EndDirItemPath();
+
+ private EndDirItemPath() {
+ }
+
+ public static EndDirItemPath getInstance(){
+ return INSTANCE;
+ }
+
+ /**
+ * When we process this Item, the context goes back up a level
+ */
+ @Override
+ public void process(Stack> stack, ItemContext ctx) {
+ Path parent = ctx.getContext().getParent();
+ ctx.setContext(parent);
+ }
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/ItemContextPath.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/ItemContextPath.java
new file mode 100644
index 000000000..c90e241ba
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/ItemContextPath.java
@@ -0,0 +1,8 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.path;
+
+import java.nio.file.Path;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+
+public class ItemContextPath extends ItemContext {
+
+}
diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/StackProcessorTest.java b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/StackProcessorTest.java
new file mode 100644
index 000000000..8a7f7cab8
--- /dev/null
+++ b/datavault-broker/src/test/java/org/datavaultplatform/common/storage/impl/ssh/stack/path/StackProcessorTest.java
@@ -0,0 +1,40 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.nio.file.Path;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.impl.ssh.stack.StackProcessor;
+import org.junit.jupiter.api.Test;
+
+/*
+This tests that we can crawl a tree of files, calculating the total size
+by using an explicit stack and not using recursion.
+ */
+@Slf4j
+public class StackProcessorTest extends BaseNestedFilesTest {
+
+ @Test
+ void testCrawlFilesUsingStackProcessor() {
+ createFiles(this.baseDir, 10);
+ assertEquals(getExpectedFileCountAtLevel(10), this.fileCounter);
+
+ ItemContextPath ctx = new ItemContextPath();
+ ctx.setContext(this.baseDir.toPath().getParent());
+
+ DirEntryPath initialItem = new DirEntryPath(this.baseDir);
+
+ StackProcessor stackProcessor = new StackProcessor<>(
+ ctx, initialItem);
+
+ stackProcessor.process();
+ long size = ctx.getSize();
+ long count = ctx.getCount();
+
+ log.info("COUNT [{}]", count);
+ log.info("SIZE [{}]", size);
+
+ assertEquals(count, fileCounter);
+ assertEquals(3 * count, size);
+ }
+}
diff --git a/datavault-broker/src/test/resources/sftpsize/createFileTree.sh b/datavault-broker/src/test/resources/sftpsize/createFileTree.sh
new file mode 100755
index 000000000..03e6c838d
--- /dev/null
+++ b/datavault-broker/src/test/resources/sftpsize/createFileTree.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# This is used to create files in a SFTP Container.
+# These files are used to test SFTPFileSystemDriver.getSize
+# See CombinedSizeSFTPJSchIT
+WD=$1
+DEPTH=$2
+
+echo "WorkingDir is [$WD]"
+echo "Depth is [$DEPTH]"
+FILES_CREATED=0
+function createFiles() {
+ local base=$1
+ local depth=$2
+ #echo "in createFiles ${base} ${depth}"
+
+ echo "aaa" > $base/a.txt
+ #ls -l $base/a.txt
+ echo "bbb" > $base/b.txt
+ #ls -l $base/b.txt
+ FILES_CREATED=$((FILES_CREATED+2))
+ if (( $depth > 0 )); then
+ local left="${base}/L"
+ local right="${base}/R"
+ local nextDepth=$((depth-1))
+ mkdir -p $left
+ createFiles $left $nextDepth
+ mkdir -p $right
+ createFiles $right $nextDepth
+ fi
+}
+createFiles $WD $DEPTH
+echo "FILES_CREATED ${FILES_CREATED}"
\ No newline at end of file
diff --git a/datavault-common/pom.xml b/datavault-common/pom.xml
index c8421fb78..4e925b459 100644
--- a/datavault-common/pom.xml
+++ b/datavault-common/pom.xml
@@ -88,7 +88,7 @@
com.oracle.oci.sdk
oci-java-sdk-objectstorage
-
+
org.bouncycastle
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/SFTPFileSystemJSch.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/SFTPFileSystemJSch.java
index 4b07bee51..96f4116ae 100644
--- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/SFTPFileSystemJSch.java
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/SFTPFileSystemJSch.java
@@ -2,7 +2,7 @@
import com.jcraft.jsch.*;
import java.io.InputStream;
-import java.nio.file.Paths;
+
import java.time.Clock;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@@ -20,6 +20,9 @@
import java.util.List;
import java.util.Map;
import java.util.Vector;
+import org.datavaultplatform.common.storage.impl.ssh.UtilityJSchImproved;
+import org.datavaultplatform.common.storage.impl.ssh.UtilityJSchNonRecurse;
+import org.springframework.util.Assert;
/**
* An implementation of SFTPFileSystemDriver to use JCraft's Jsch ssh/sftp library.
@@ -259,9 +262,33 @@ public long getSize(String path) throws Exception {
if (!path.endsWith("/")) {
path = path + "/";
}
-
- return UtilityJSch.calculateSize(channelSftp, path);
-
+
+ Runtime.getRuntime().gc();
+ long t5 = System.currentTimeMillis();
+ long sizeNonRecurse = UtilityJSchNonRecurse.calculateSize(channelSftp, path);
+ long t6 = System.currentTimeMillis();
+ long timeNonRecurse = t6 - t5;
+ log.info("NON RECURSE TIME [{}]", timeNonRecurse);
+
+ Runtime.getRuntime().gc();
+ long t3 = System.currentTimeMillis();
+ long sizeImproved = UtilityJSchImproved.calculateSize(channelSftp, path);
+ long t4 = System.currentTimeMillis();
+ long timeImproved = t4 - t3;
+ log.info("IMPROVED TIME [{}]", timeImproved);
+
+ Runtime.getRuntime().gc();
+ long t1 = System.currentTimeMillis();
+ long sizeOriginal = UtilityJSch.calculateSize(channelSftp, path);
+ long t2 = System.currentTimeMillis();
+ long timeOriginal = t2 - t1;
+ log.info("ORIG TIME [{}]", timeOriginal);
+
+ Assert.isTrue(sizeOriginal == sizeImproved,
+ String.format("original size[%s] != improved size[%s]", sizeOriginal, sizeImproved));
+ Assert.isTrue(sizeOriginal == sizeNonRecurse,
+ String.format("original size[%s] != non-recurse size[%s]", sizeOriginal, sizeNonRecurse));
+ return sizeOriginal;
} else {
return attrs.getSize();
}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchImproved.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchImproved.java
new file mode 100644
index 000000000..b6dc2cfa4
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchImproved.java
@@ -0,0 +1,60 @@
+package org.datavaultplatform.common.storage.impl.ssh;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import com.jcraft.jsch.SftpException;
+import java.util.Comparator;
+import java.util.Iterator;
+
+public abstract class UtilityJSchImproved {
+
+ public static long calculateSize(final ChannelSftp channel,
+ String remoteFile) throws SftpException {
+
+ long bytes = 0;
+ String pwd = remoteFile;
+
+ if (remoteFile.lastIndexOf('/') != -1) {
+ if (remoteFile.length() > 1) {
+ pwd = remoteFile.substring(0, remoteFile.lastIndexOf('/'));
+ }
+ }
+
+ channel.cd(pwd);
+
+ @SuppressWarnings("unchecked") final java.util.Vector files = channel.ls(remoteFile);
+ // Sort the entries, we will process directories last.
+ files.sort(COMPARATOR);
+ Iterator iter = files.iterator();
+ //process each entry, removing it from Vector after processing, go to aid GC.
+ while(iter.hasNext()) {
+ final ChannelSftp.LsEntry le = iter.next();
+
+ // remove items from the Vector as soon as possible
+ iter.remove();
+
+ final String name = le.getFilename();
+ if (le.getAttrs().isDir()) {
+ if (name.equals(".") || name.equals("..")) {
+ continue;
+ }
+ bytes += calculateSize(channel, channel.pwd() + "/" + name + "/");
+ } else {
+ bytes += le.getAttrs().getSize();
+ }
+ }
+
+ channel.cd("..");
+ return bytes;
+ }
+
+ private static final Comparator COMPARATOR_DIR =
+ Comparator.comparing(entry -> entry.getAttrs().isDir());
+
+ private static final Comparator COMPARATOR_FILENAME =
+ Comparator.comparing(entry -> entry.getFilename());
+
+ public static final Comparator COMPARATOR =
+ COMPARATOR_DIR.thenComparing(COMPARATOR_FILENAME);
+
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchNonRecurse.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchNonRecurse.java
new file mode 100644
index 000000000..07a824445
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchNonRecurse.java
@@ -0,0 +1,58 @@
+package org.datavaultplatform.common.storage.impl.ssh;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import com.jcraft.jsch.SftpException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Vector;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.impl.ssh.stack.jsch.ItemContextChannel;
+import org.datavaultplatform.common.storage.impl.ssh.stack.jsch.ItemLsEntryDir;
+import org.datavaultplatform.common.storage.impl.ssh.stack.jsch.ItemLsEntryNonDir;
+import org.datavaultplatform.common.storage.impl.ssh.stack.Item;
+import org.datavaultplatform.common.storage.impl.ssh.stack.StackProcessor;
+
+@Slf4j
+public abstract class UtilityJSchNonRecurse {
+
+ public static long calculateSize(final ChannelSftp channel,
+ String remoteFile) throws SftpException {
+
+ if (remoteFile.lastIndexOf('/') != -1) {
+ if (remoteFile.length() > 1) {
+ remoteFile = remoteFile.substring(0, remoteFile.lastIndexOf('/'));
+ }
+ }
+ channel.cd(remoteFile);
+ channel.cd("..");
+
+ ItemContextChannel ctx = new ItemContextChannel(channel);
+
+ LsEntry initialEntry = getInitialEntry(channel, remoteFile);
+
+ Item initialItem =
+ initialEntry.getAttrs().isDir()
+ ? new ItemLsEntryDir(initialEntry)
+ : new ItemLsEntryNonDir(initialEntry);
+
+ StackProcessor processor = new StackProcessor<>(ctx, initialItem);
+ processor.process();
+ return ctx.getSize();
+ }
+
+ @SneakyThrows
+ private static LsEntry getInitialEntry(ChannelSftp channel, String remoteFile) {
+ Path path = Paths.get(remoteFile);
+ int count = path.getNameCount();
+ String endPath = path.getName(count - 1).toString();
+
+ Vector entries = channel.ls("*");
+ LsEntry initialEntry = entries.stream()
+ .filter((LsEntry entry) -> endPath.equals(entry.getFilename()))
+ .findFirst()
+ .get();
+ return initialEntry;
+ }
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/Item.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/Item.java
new file mode 100644
index 000000000..0485217be
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/Item.java
@@ -0,0 +1,8 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack;
+
+import java.util.Stack;
+
+public interface Item {
+
+ void process(Stack> stack, ItemContext context);
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/ItemContext.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/ItemContext.java
new file mode 100644
index 000000000..f28c303e9
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/ItemContext.java
@@ -0,0 +1,29 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack;
+
+public abstract class ItemContext {
+
+ private T ctx;
+ private long size;
+ private long count;
+
+ public long getSize(){
+ return size;
+ }
+ public T getContext() {
+ return ctx;
+ }
+
+ public void setContext(T ctx) {
+ this.ctx = ctx;
+ }
+
+ public void increment(long inc){
+ size += inc;
+ }
+ public long getCount(){
+ return count;
+ }
+ public void incrementCount(){
+ count+=1;
+ }
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/StackProcessor.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/StackProcessor.java
new file mode 100644
index 000000000..b72e78849
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/StackProcessor.java
@@ -0,0 +1,40 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack;
+
+import java.util.Stack;
+
+/*
+An abstract stack processor which can be used to crawl a tree of directories/files
+without using recursion.
+@See org.datavaultplatform.common.storage.impl.ssh.UtilityJSchNonRecurse
+@see
+ */
+public class StackProcessor {
+
+ private final ItemContext context;
+ private final Item initialItem;
+
+ public StackProcessor(ItemContext context, Item initialItem) {
+ this.context = context;
+ this.initialItem = initialItem;
+ }
+
+ /*
+ when crawling directories, the Stack represents files/directories discovered but not processed
+ and the context can represent the current directory and any calculated results
+ */
+ public ItemContext process() {
+
+
+ Stack> stack = new Stack();
+
+ stack.push(initialItem);
+
+ while (!stack.isEmpty()) {
+ Item item = stack.pop();
+ // when we process an item it may push more items on the stack, it may affect the context
+ item.process(stack, context);
+ }
+ return context;
+ }
+}
+
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemContextChannel.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemContextChannel.java
new file mode 100644
index 000000000..e973b75f7
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemContextChannel.java
@@ -0,0 +1,17 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import com.jcraft.jsch.ChannelSftp;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+
+@Slf4j
+public class ItemContextChannel extends ItemContext {
+
+ @SneakyThrows
+ public ItemContextChannel(ChannelSftp channel){
+ log.info("initial pwd [{}]", channel.pwd());
+ setContext(channel);
+ }
+
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryDir.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryDir.java
new file mode 100644
index 000000000..9a30061d5
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryDir.java
@@ -0,0 +1,46 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.Vector;
+import java.util.stream.Collectors;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.impl.ssh.stack.Item;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+import org.springframework.util.Assert;
+
+@Slf4j
+public class ItemLsEntryDir implements Item {
+
+ private final LsEntry listEntry;
+
+ public ItemLsEntryDir(LsEntry listEntry) {
+ Assert.isTrue(listEntry.getAttrs().isDir());
+ this.listEntry = listEntry;
+ }
+
+ @Override
+ @SneakyThrows
+ public void process(Stack> stack, ItemContext ctx) {
+ ctx.getContext().cd(this.listEntry.getFilename());
+
+ Vector entries = ctx.getContext().ls("*");
+
+ Map> groupedItems = entries.stream()
+ .collect(Collectors.partitioningBy((LsEntry item) -> item.getAttrs().isDir()));
+
+ // push 'cd ..' - process this third (last)
+ stack.push(ItemLsEntryEndDir.getInstance());
+
+ // push all directories - process these second
+ groupedItems.get(true).forEach(dirEntry -> stack.push(new ItemLsEntryDir(dirEntry)));
+
+ // push all 'non directories' - process these first
+ groupedItems.get(false).forEach(nonDirEntry -> stack.push(new ItemLsEntryNonDir(nonDirEntry)));
+ }
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryEndDir.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryEndDir.java
new file mode 100644
index 000000000..10cf9a5b6
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryEndDir.java
@@ -0,0 +1,29 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import java.util.Stack;
+import lombok.SneakyThrows;
+import org.datavaultplatform.common.storage.impl.ssh.stack.Item;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+
+public class ItemLsEntryEndDir implements Item {
+
+ private static final ItemLsEntryEndDir INSTANCE = new ItemLsEntryEndDir();
+
+ private ItemLsEntryEndDir() {
+ }
+
+ public static ItemLsEntryEndDir getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * When we process this Item, the context goes back up a level
+ */
+ @Override
+ @SneakyThrows
+ public void process(Stack> stack, ItemContext ctx) {
+ ctx.getContext().cd("..");
+ }
+}
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryNonDir.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryNonDir.java
new file mode 100644
index 000000000..6ae757a18
--- /dev/null
+++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/ssh/stack/jsch/ItemLsEntryNonDir.java
@@ -0,0 +1,27 @@
+package org.datavaultplatform.common.storage.impl.ssh.stack.jsch;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import java.util.Stack;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.storage.impl.ssh.stack.Item;
+import org.datavaultplatform.common.storage.impl.ssh.stack.ItemContext;
+import org.springframework.util.Assert;
+
+@Slf4j
+public class ItemLsEntryNonDir implements Item {
+
+ private final LsEntry listEntry;
+
+ public ItemLsEntryNonDir(LsEntry listEntry) {
+ Assert.isTrue(!listEntry.getAttrs().isDir());
+ this.listEntry = listEntry;
+ }
+
+ @Override
+ @SneakyThrows
+ public void process(Stack> stack, ItemContext ctx) {
+ ctx.increment(this.listEntry.getAttrs().getSize());
+ }
+}
diff --git a/datavault-common/src/test/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchImprovedTest.java b/datavault-common/src/test/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchImprovedTest.java
new file mode 100644
index 000000000..4a5827882
--- /dev/null
+++ b/datavault-common/src/test/java/org/datavaultplatform/common/storage/impl/ssh/UtilityJSchImprovedTest.java
@@ -0,0 +1,88 @@
+package org.datavaultplatform.common.storage.impl.ssh;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.when;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import com.jcraft.jsch.SftpATTRS;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Vector;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/**
+ * This is a test of
+ * 1) the Comparator used in UtilityJSchImproved
+ * 2) the removal of items from an underlying Vector using Iterator.remove()
+ */
+@ExtendWith(MockitoExtension.class)
+public class UtilityJSchImprovedTest {
+
+ @Mock
+ ChannelSftp.LsEntry entry1;
+ @Mock
+ ChannelSftp.LsEntry entry2;
+ @Mock
+ ChannelSftp.LsEntry entry3;
+ @Mock
+ ChannelSftp.LsEntry entry4;
+
+ @Mock
+ SftpATTRS attrs1;
+
+ @Mock
+ SftpATTRS attrs2;
+ @Mock
+ SftpATTRS attrs3;
+ @Mock
+ SftpATTRS attrs4;
+
+ @Test
+ void testComparator() {
+
+ when(entry1.getAttrs()).thenReturn(attrs1);
+ when(entry2.getAttrs()).thenReturn(attrs2);
+ when(entry3.getAttrs()).thenReturn(attrs3);
+ when(entry4.getAttrs()).thenReturn(attrs4);
+
+ when(attrs1.isDir()).thenReturn(true);
+ when(attrs2.isDir()).thenReturn(true);
+ when(attrs3.isDir()).thenReturn( false);
+ when(attrs4.isDir()).thenReturn(false);
+ when(entry1.getFilename()).thenReturn("D-BBBB");
+ when(entry2.getFilename()).thenReturn("D-AAAA");
+ when(entry3.getFilename()).thenReturn("F-BBBB");
+ when(entry4.getFilename()).thenReturn("F-AAAA");
+
+ List items = Arrays.asList(this.entry1, this.entry2, this.entry3, this.entry4);
+ Collections.shuffle(items);
+
+ items.forEach(item -> System.out.printf("BEFORE[%s]%n",item.getFilename()));
+ items.sort(UtilityJSchImproved.COMPARATOR);
+
+ items.forEach(item -> System.out.printf("AFTER[%s]%n",item.getFilename()));
+
+ assertEquals(entry4, items.get(0));
+ assertEquals(entry3, items.get(1));
+ assertEquals(entry2, items.get(2));
+ assertEquals(entry1, items.get(3));
+ }
+
+ @Test
+ void testRemoveFromVectorIterator(){
+ Vector items = new Vector<>(Arrays.asList("1","2","3","4","5"));
+ Iterator iter = items.iterator();
+ while(iter.hasNext()) {
+ String item = iter.next();
+ System.out.printf("item[%s]items count[%d]%n", item, items.size());
+ iter.remove();
+ }
+ assertEquals(0, items.size());
+ }
+}
diff --git a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibUtils.java b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/authorization/AuthorizationUtils.java
similarity index 84%
rename from datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibUtils.java
rename to datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/authorization/AuthorizationUtils.java
index d50a1bdee..9656a210c 100644
--- a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibUtils.java
+++ b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/authorization/AuthorizationUtils.java
@@ -1,10 +1,10 @@
-package org.datavaultplatform.webapp.authentication.shib;
+package org.datavaultplatform.webapp.authentication.authorization;
import org.datavaultplatform.common.model.RoleName;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
-public abstract class ShibUtils {
+public abstract class AuthorizationUtils {
public static final GrantedAuthority ROLE_USER = new SimpleGrantedAuthority(RoleName.ROLE_USER);
public static final GrantedAuthority ROLE_ADMIN = new SimpleGrantedAuthority(RoleName.ROLE_ADMIN);
public static final GrantedAuthority ROLE_IS_ADMIN = new SimpleGrantedAuthority(RoleName.ROLE_IS_ADMIN);
diff --git a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibGrantedAuthorityService.java b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/authorization/GrantedAuthorityService.java
similarity index 88%
rename from datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibGrantedAuthorityService.java
rename to datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/authorization/GrantedAuthorityService.java
index aacc2af51..3011dcab9 100644
--- a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibGrantedAuthorityService.java
+++ b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/authorization/GrantedAuthorityService.java
@@ -1,4 +1,4 @@
-package org.datavaultplatform.webapp.authentication.shib;
+package org.datavaultplatform.webapp.authentication.authorization;
import java.security.Principal;
import java.util.ArrayList;
@@ -17,13 +17,13 @@
import org.springframework.security.core.authority.SimpleGrantedAuthority;
@Slf4j
-public class ShibGrantedAuthorityService {
+public class GrantedAuthorityService {
private final RestService restService;
private final PermissionsService permissionsService;
- public ShibGrantedAuthorityService(RestService restService, PermissionsService permissionsService) {
+ public GrantedAuthorityService(RestService restService, PermissionsService permissionsService) {
this.restService = restService;
this.permissionsService = permissionsService;
}
@@ -40,7 +40,7 @@ public List getGrantedAuthoritiesForUser(String name, Authenti
try {
isAdmin = restService.isAdmin(new ValidateUser(name, null));
if (isAdmin) {
- grantedAuths.add(ShibUtils.ROLE_IS_ADMIN);
+ grantedAuths.add(AuthorizationUtils.ROLE_IS_ADMIN);
}
} catch (Exception ex) {
log.error("Error when trying to check if user is admin with Broker!", ex);
@@ -49,7 +49,7 @@ public List getGrantedAuthoritiesForUser(String name, Authenti
Collection adminAuthorities = getAdminAuthorities(authentication);
if (!adminAuthorities.isEmpty()) {
log.info("Granting user {} ROLE_ADMIN", name);
- grantedAuths.add(ShibUtils.ROLE_ADMIN);
+ grantedAuths.add(AuthorizationUtils.ROLE_ADMIN);
grantedAuths.addAll(adminAuthorities);
}
return grantedAuths;
diff --git a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/database/DatabaseAuthenticationProvider.java b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/database/DatabaseAuthenticationProvider.java
index f8fef46c7..3a6c89dde 100644
--- a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/database/DatabaseAuthenticationProvider.java
+++ b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/database/DatabaseAuthenticationProvider.java
@@ -1,11 +1,8 @@
package org.datavaultplatform.webapp.authentication.database;
-import org.datavaultplatform.common.model.RoleAssignment;
import org.datavaultplatform.common.model.RoleName;
import org.datavaultplatform.common.request.ValidateUser;
-import org.datavaultplatform.webapp.security.ScopedGrantedAuthority;
-import org.datavaultplatform.webapp.model.AdminDashboardPermissionsModel;
-import org.datavaultplatform.webapp.services.PermissionsService;
+import org.datavaultplatform.webapp.authentication.authorization.GrantedAuthorityService;
import org.datavaultplatform.webapp.services.RestService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -17,11 +14,8 @@
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
-import java.security.Principal;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
-import java.util.stream.Collectors;
/**
* User: Robin Taylor
@@ -35,11 +29,11 @@ public class DatabaseAuthenticationProvider implements AuthenticationProvider {
private final RestService restService;
- private final PermissionsService permissionsService;
+ private final GrantedAuthorityService grantedAuthorityService;
- public DatabaseAuthenticationProvider(RestService restService, PermissionsService permissionsService) {
+ public DatabaseAuthenticationProvider(RestService restService, GrantedAuthorityService grantedAuthorityService) {
this.restService = restService;
- this.permissionsService = permissionsService;
+ this.grantedAuthorityService = grantedAuthorityService;
}
@Override
@@ -63,40 +57,13 @@ public Authentication authenticate(Authentication authentication) throws Authent
logger.info("Authentication success for " + name);
List grantedAuths = new ArrayList<>();
- List roles = restService.getRoleAssignmentsForUser(name);
- List scopedAuthorities = ScopedGrantedAuthority.fromRoleAssignments(roles);
-
- grantedAuths.addAll(scopedAuthorities);
-
- boolean isAdmin = false;
- try{
- isAdmin = restService.isAdmin(new ValidateUser(name, null));
- if (isAdmin) {
- grantedAuths.add(new SimpleGrantedAuthority(RoleName.ROLE_IS_ADMIN));
- }
- } catch(Exception e){
- logger.error("Error when trying to check if user is admin with Broker!",e);
- }
- Collection adminAuthorities = getAdminAuthorities(authentication);
- if (!adminAuthorities.isEmpty()) {
- logger.info("Granting user " + name + " " + RoleName.ROLE_ADMIN);
- grantedAuths.add(new SimpleGrantedAuthority(RoleName.ROLE_ADMIN));
- grantedAuths.addAll(adminAuthorities);
- }
+ List grantedAuthsForUser = this.grantedAuthorityService.getGrantedAuthoritiesForUser(name, authentication);
+ grantedAuths.addAll(grantedAuthsForUser);
logger.info("Granting user " + name + " " + RoleName.ROLE_USER);
grantedAuths.add(new SimpleGrantedAuthority(RoleName.ROLE_USER));
return new UsernamePasswordAuthenticationToken(name, password, grantedAuths);
-
- }
-
- private Collection getAdminAuthorities(Principal principal) {
- AdminDashboardPermissionsModel adminPermissions = permissionsService.getDashboardPermissions(principal);
- return adminPermissions.getUnscopedPermissions().stream()
- .filter(p -> p.getPermission().getRoleName() != null)
- .map(p -> new SimpleGrantedAuthority(p.getPermission().getRoleName()))
- .collect(Collectors.toSet());
}
@Override
diff --git a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibAuthenticationProvider.java b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibAuthenticationProvider.java
index 5cd5d1eec..3be03c166 100644
--- a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibAuthenticationProvider.java
+++ b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/authentication/shib/ShibAuthenticationProvider.java
@@ -4,6 +4,8 @@
import org.datavaultplatform.common.request.ValidateUser;
import org.datavaultplatform.common.services.LDAPService;
import org.datavaultplatform.webapp.authentication.AuthenticationSuccess;
+import org.datavaultplatform.webapp.authentication.authorization.AuthorizationUtils;
+import org.datavaultplatform.webapp.authentication.authorization.GrantedAuthorityService;
import org.datavaultplatform.webapp.services.RestService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,18 +37,18 @@ public class ShibAuthenticationProvider implements AuthenticationProvider {
private final RestService restService;
private final LDAPService ldapService;
- private final ShibGrantedAuthorityService shibGrantedAuthorityService;
+ private final GrantedAuthorityService grantedAuthorityService;
private final boolean ldapEnabled;
@Autowired
private AuthenticationSuccess authenticationSuccess;
- public ShibAuthenticationProvider(RestService restService, LDAPService ldapService, boolean ldapEnabled, ShibGrantedAuthorityService shibGrantedAuthorityService ) {
+ public ShibAuthenticationProvider(RestService restService, LDAPService ldapService, boolean ldapEnabled, GrantedAuthorityService grantedAuthorityService) {
this.restService = restService;
this.ldapService = ldapService;
this.ldapEnabled = ldapEnabled;
- this.shibGrantedAuthorityService = shibGrantedAuthorityService;
+ this.grantedAuthorityService = grantedAuthorityService;
}
@@ -84,11 +86,11 @@ public Authentication authenticate(Authentication authentication) throws Authent
logger.error("Error when trying to add user with Broker!",e);
}
} else {
- List grantedAuthsForUser = this.shibGrantedAuthorityService.getGrantedAuthoritiesForUser(name, authentication);
+ List grantedAuthsForUser = this.grantedAuthorityService.getGrantedAuthoritiesForUser(name, authentication);
grantedAuths.addAll(grantedAuthsForUser);
}
- grantedAuths.add(ShibUtils.ROLE_USER);
+ grantedAuths.add(AuthorizationUtils.ROLE_USER);
return new PreAuthenticatedAuthenticationToken(name, password, grantedAuths);
}
diff --git a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/database/DatabaseProfileConfig.java b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/database/DatabaseProfileConfig.java
index f55270536..0875aad6f 100644
--- a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/database/DatabaseProfileConfig.java
+++ b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/database/DatabaseProfileConfig.java
@@ -1,5 +1,6 @@
package org.datavaultplatform.webapp.config.database;
+import org.datavaultplatform.webapp.authentication.authorization.GrantedAuthorityService;
import org.datavaultplatform.webapp.authentication.database.DatabaseAuthenticationProvider;
import org.datavaultplatform.webapp.services.PermissionsService;
import org.datavaultplatform.webapp.services.RestService;
@@ -15,7 +16,12 @@ public class DatabaseProfileConfig {
@Bean
DatabaseAuthenticationProvider databaseAuthenticationProvider(RestService restService, PermissionsService permissionsService){
- return new DatabaseAuthenticationProvider(restService, permissionsService);
+ return new DatabaseAuthenticationProvider(restService, grantedAuthorityService(restService,permissionsService));
+ }
+
+ @Bean
+ GrantedAuthorityService grantedAuthorityService(RestService restService, PermissionsService permissionsService) {
+ return new GrantedAuthorityService(restService, permissionsService);
}
}
diff --git a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/shib/ShibProfileConfig.java b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/shib/ShibProfileConfig.java
index 0979c9252..e343d7d24 100644
--- a/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/shib/ShibProfileConfig.java
+++ b/datavault-webapp/src/main/java/org/datavaultplatform/webapp/config/shib/ShibProfileConfig.java
@@ -4,7 +4,7 @@
import org.datavaultplatform.common.services.LDAPService;
import org.datavaultplatform.webapp.authentication.shib.ShibAuthenticationListener;
import org.datavaultplatform.webapp.authentication.shib.ShibAuthenticationProvider;
-import org.datavaultplatform.webapp.authentication.shib.ShibGrantedAuthorityService;
+import org.datavaultplatform.webapp.authentication.authorization.GrantedAuthorityService;
import org.datavaultplatform.webapp.authentication.shib.ShibWebAuthenticationDetailsSource;
import org.datavaultplatform.webapp.services.PermissionsService;
import org.datavaultplatform.webapp.services.RestService;
@@ -34,9 +34,9 @@ ShibAuthenticationProvider shibAuthenticationProvider(
RestService restService,
LDAPService ldapService,
@Value("${ldap.enabled}") boolean ldapEnabled,
- ShibGrantedAuthorityService shibGrantedAuthorityService
+ GrantedAuthorityService grantedAuthorityService
){
- ShibAuthenticationProvider result = new ShibAuthenticationProvider(restService, ldapService, ldapEnabled, shibGrantedAuthorityService);
+ ShibAuthenticationProvider result = new ShibAuthenticationProvider(restService, ldapService, ldapEnabled, grantedAuthorityService);
return result;
}
@@ -67,8 +67,8 @@ Http403ForbiddenEntryPoint http403EntryPoint() {
}
@Bean
- ShibGrantedAuthorityService shibGrantedAuthorityService(RestService restService, PermissionsService permissionsService) {
- return new ShibGrantedAuthorityService(restService, permissionsService);
+ GrantedAuthorityService shibGrantedAuthorityService(RestService restService, PermissionsService permissionsService) {
+ return new GrantedAuthorityService(restService, permissionsService);
}
@Bean
diff --git a/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/ajp/AjpConnectorIT.java b/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/ajp/AjpConnectorIT.java
index 93557ce42..9fe9d3815 100644
--- a/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/ajp/AjpConnectorIT.java
+++ b/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/ajp/AjpConnectorIT.java
@@ -19,7 +19,7 @@
import org.datavaultplatform.common.services.LDAPService;
import org.datavaultplatform.common.util.DisabledInsideDocker;
import org.datavaultplatform.webapp.authentication.shib.ShibAuthenticationListener;
-import org.datavaultplatform.webapp.authentication.shib.ShibGrantedAuthorityService;
+import org.datavaultplatform.webapp.authentication.authorization.GrantedAuthorityService;
import org.datavaultplatform.webapp.services.RestService;
import org.datavaultplatform.webapp.test.ProfileShib;
import org.junit.jupiter.api.BeforeEach;
@@ -72,7 +72,7 @@ class AjpConnectorIT {
LDAPService mLdapService;
@MockBean
- ShibGrantedAuthorityService mGrantedAuthorityService;
+ GrantedAuthorityService mGrantedAuthorityService;
@Mock
User mUser;
diff --git a/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/authentication/shib/LoginUsingShibTest.java b/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/authentication/shib/LoginUsingShibTest.java
index 1aa4a0654..6b3b94234 100644
--- a/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/authentication/shib/LoginUsingShibTest.java
+++ b/datavault-webapp/src/test/java/org/datavaultplatform/webapp/app/authentication/shib/LoginUsingShibTest.java
@@ -25,8 +25,8 @@
import org.datavaultplatform.common.request.ValidateUser;
import org.datavaultplatform.common.services.LDAPService;
import org.datavaultplatform.webapp.authentication.shib.ShibAuthenticationListener;
-import org.datavaultplatform.webapp.authentication.shib.ShibGrantedAuthorityService;
-import org.datavaultplatform.webapp.authentication.shib.ShibUtils;
+import org.datavaultplatform.webapp.authentication.authorization.GrantedAuthorityService;
+import org.datavaultplatform.webapp.authentication.authorization.AuthorizationUtils;
import org.datavaultplatform.webapp.authentication.shib.ShibWebAuthenticationDetails;
import org.datavaultplatform.webapp.services.RestService;
import org.datavaultplatform.webapp.test.ProfileShib;
@@ -70,7 +70,7 @@ public class LoginUsingShibTest {
LDAPService mLdapService;
@MockBean
- ShibGrantedAuthorityService mGrantedAuthorityService;
+ GrantedAuthorityService mGrantedAuthorityService;
@Mock
GrantedAuthority mGA1;
@@ -124,7 +124,7 @@ void testShibLoginExistingUser() throws Exception {
assertEquals("N/A", argAuthentication.getValue().getCredentials());
Mockito.verify(mGrantedAuthorityService).getGrantedAuthoritiesForUser(argName.getValue(), argAuthentication.getValue());
- checkLoggedInUser(result, sessionId, new HashSet<>(Arrays.asList(ShibUtils.ROLE_USER, mGA1, mGA2)));
+ checkLoggedInUser(result, sessionId, new HashSet<>(Arrays.asList(AuthorizationUtils.ROLE_USER, mGA1, mGA2)));
Mockito.verify(mAuthListener).onApplicationEvent(argAuthSuccessEvent.getValue());
PreAuthenticatedAuthenticationToken auth = (PreAuthenticatedAuthenticationToken)argAuthSuccessEvent.getValue().getAuthentication();
@@ -171,7 +171,7 @@ void testShibLoginNewUser() throws Exception {
assertNull(argUser.getValue().getFileStores());
assertNull(argUser.getValue().getVaults());
- checkLoggedInUser(result, sessionId, Collections.singleton(ShibUtils.ROLE_USER));
+ checkLoggedInUser(result, sessionId, Collections.singleton(AuthorizationUtils.ROLE_USER));
Mockito.verify(mAuthListener).onApplicationEvent(argAuthSuccessEvent.getValue());
@@ -180,7 +180,7 @@ void testShibLoginNewUser() throws Exception {
}
void checkLoggedInUser(MvcResult result, String expectedSessionId, Set expectedAuthorities){
- SecurityContext sc = (SecurityContext) result.getRequest().getSession().getAttribute(ShibUtils.SPRING_SECURITY_CONTEXT);
+ SecurityContext sc = (SecurityContext) result.getRequest().getSession().getAttribute(AuthorizationUtils.SPRING_SECURITY_CONTEXT);
Authentication auth = sc.getAuthentication();
assertTrue(auth instanceof PreAuthenticatedAuthenticationToken);
diff --git a/datavault-worker/src/test/java/org/datavaultplatform/broker/services/UserKeyPairService.java b/datavault-worker/src/test/java/org/datavaultplatform/broker/services/UserKeyPairService.java
new file mode 100644
index 000000000..a39fc94c4
--- /dev/null
+++ b/datavault-worker/src/test/java/org/datavaultplatform/broker/services/UserKeyPairService.java
@@ -0,0 +1,24 @@
+package org.datavaultplatform.broker.services;
+
+import lombok.Builder;
+import lombok.Data;
+
+public interface UserKeyPairService {
+
+ // comment added at the end of public key
+ String PUBKEY_COMMENT = "datavault";
+
+ String getPassphrase();
+
+ KeyPairInfo generateNewKeyPair();
+
+ @Data
+ @Builder
+ class KeyPairInfo {
+
+ private final String publicKey;
+ private final String privateKey;
+ private final String fingerPrint;
+ private final Integer keySize;
+ }
+}
diff --git a/datavault-worker/src/test/java/org/datavaultplatform/broker/services/UserKeyPairServiceImpl.java b/datavault-worker/src/test/java/org/datavaultplatform/broker/services/UserKeyPairServiceImpl.java
new file mode 100644
index 000000000..f5be5ec36
--- /dev/null
+++ b/datavault-worker/src/test/java/org/datavaultplatform/broker/services/UserKeyPairServiceImpl.java
@@ -0,0 +1,71 @@
+package org.datavaultplatform.broker.services;
+
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.common.crypto.Encryption;
+import org.datavaultplatform.common.crypto.SshRsaKeyUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.interfaces.RSAPrivateKey;
+import java.security.interfaces.RSAPublicKey;
+
+/**
+ * User: David Hay
+ * Date: 22/Aug/2022
+ * Time: 16:45
+ */
+@Service
+@Slf4j
+@Transactional
+@Primary
+public class UserKeyPairServiceImpl implements UserKeyPairService {
+
+ static {
+ Encryption.addBouncyCastleSecurityProvider();
+ }
+
+ private final String passphrase;
+
+ private static final int KEY_SIZE = 1024;
+
+ @Autowired
+ public UserKeyPairServiceImpl(@Value("${sftp.passphrase}") String passphrase) {
+ this.passphrase = passphrase;
+ }
+
+ @Override
+ public String getPassphrase() {
+ return passphrase;
+ }
+
+ @Override
+ @SneakyThrows
+ public KeyPairInfo generateNewKeyPair() {
+
+ KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
+ generator.initialize(1024);
+
+ KeyPair kp = generator.generateKeyPair();
+ RSAPublicKey publicKey = (RSAPublicKey) kp.getPublic();
+ RSAPrivateKey privateKey = (RSAPrivateKey) kp.getPrivate();
+
+ String privateKeyValue = SshRsaKeyUtils.encodePrivateKey(privateKey, passphrase);
+
+ String publicKeyValue = SshRsaKeyUtils.encodePublicKey(publicKey, PUBKEY_COMMENT);
+
+ return KeyPairInfo.builder()
+ .privateKey(privateKeyValue)
+ .publicKey(publicKeyValue)
+ .fingerPrint(SshRsaKeyUtils.calculateFingerprint(publicKeyValue))
+ .keySize(KEY_SIZE)
+ .build();
+
+ }
+}
diff --git a/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/BasePerformDepositThenRetrieveUsingSftpIT.java b/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/BasePerformDepositThenRetrieveUsingSftpIT.java
new file mode 100644
index 000000000..5ea5a1347
--- /dev/null
+++ b/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/BasePerformDepositThenRetrieveUsingSftpIT.java
@@ -0,0 +1,560 @@
+package org.datavaultplatform.worker.tasks.sftp;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.rabbitmq.client.Channel;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.awaitility.Awaitility;
+import org.datavaultplatform.broker.services.UserKeyPairService;
+import org.datavaultplatform.broker.services.UserKeyPairServiceImpl;
+import org.datavaultplatform.common.PropNames;
+import org.datavaultplatform.common.config.BaseQueueConfig;
+import org.datavaultplatform.common.crypto.Encryption;
+import org.datavaultplatform.common.docker.DockerImage;
+import org.datavaultplatform.common.event.Event;
+import org.datavaultplatform.common.event.deposit.Complete;
+import org.datavaultplatform.common.event.deposit.ComputedDigest;
+import org.datavaultplatform.common.event.deposit.ComputedEncryption;
+import org.datavaultplatform.common.event.retrieve.RetrieveComplete;
+import org.datavaultplatform.common.io.FileUtils;
+import org.datavaultplatform.common.storage.Verify;
+import org.datavaultplatform.common.storage.impl.SFTPFileSystem;
+import org.datavaultplatform.common.task.Context.AESMode;
+import org.datavaultplatform.common.util.StorageClassNameResolver;
+import org.datavaultplatform.worker.rabbit.BaseRabbitTCTest;
+import org.datavaultplatform.worker.tasks.Deposit;
+import org.datavaultplatform.worker.utils.DepositEvents;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.amqp.core.AmqpAdmin;
+import org.springframework.amqp.core.Message;
+import org.springframework.amqp.core.MessageProperties;
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.support.AmqpHeaders;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+import org.springframework.messaging.handler.annotation.Header;
+import org.springframework.test.context.DynamicPropertyRegistry;
+import org.springframework.test.context.DynamicPropertySource;
+import org.springframework.test.context.TestPropertySource;
+import org.testcontainers.containers.Container;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
+
+import javax.crypto.SecretKey;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.*;
+
+@Slf4j
+@TestPropertySource(properties = {"chunking.enabled=true","chunking.size=20MB"})
+public abstract class BasePerformDepositThenRetrieveUsingSftpIT extends BaseRabbitTCTest {
+
+ private static final String TEST_USER = "testuser";
+ private static final String ENV_USER_NAME = "USER_NAME";
+ private static final String ENV_PUBLIC_KEY = "PUBLIC_KEY";
+ private static final String TEST_PASSPHRASE = "tenet";
+
+ static final String KEY_NAME_FOR_SSH = "key-name-for-ssh";
+ static final String KEY_NAME_FOR_DATA = "key-name-for-data";
+ static final String KEY_STORE_PASSWORD = "testPassword";
+
+ final Resource depositMessage = new ClassPathResource("sampleMessages/sampleDepositMessage.json");
+
+ final List events = new ArrayList<>();
+ @Autowired
+ protected AmqpAdmin rabbitAdmin;
+ @Autowired
+ protected RabbitTemplate template;
+ @Autowired
+ @Qualifier("workerQueue") //the name of the bean, not the Q
+ protected Queue workerQueue;
+ String keyStorePath;
+
+ @Value("${metaDir}")
+ String metaDir;
+ File sourceDir;
+ File destDir;
+ File retrieveBaseDir;
+ File retrieveDir;
+ @Autowired
+ ObjectMapper mapper;
+
+ @Value("classpath:big_data/50MB_file")
+ Resource largeFile;
+
+ @Value("${chunking.enabled:false}")
+ private boolean chunkingEnabled;
+
+ @Value("${chunking.size:0}")
+ private String chunkingByteSize;
+
+ @Autowired
+ private StorageClassNameResolver resolver;
+
+ private String sftpPublicKey;
+ private String sftpPrivateKey;
+
+ private GenericContainer userDataSourceContainer;
+
+ private HashMap sftpSrcProps;
+ private HashMap sftpTargetProps;
+
+ @SneakyThrows
+ static Set getPathsWithinTarFile(File tarFile) {
+ Set paths = new HashSet<>();
+ try (TarArchiveInputStream tarIn = new TarArchiveInputStream(Files.newInputStream(tarFile.toPath()))) {
+ TarArchiveEntry entry;
+ while ((entry = tarIn.getNextTarEntry()) != null) {
+ if (entry.isDirectory()) {
+ continue;
+ }
+ paths.add(Paths.get(entry.getName()));
+ }
+ }
+ return paths;
+ }
+
+ @DynamicPropertySource
+ @SneakyThrows
+ static void setupProperties(DynamicPropertyRegistry registry) {
+ File baseTemp = Files.createTempDirectory("test").toFile();
+ File tempDir = new File(baseTemp, "temp");
+ assertTrue(tempDir.mkdir());
+
+ File metaDir = new File(baseTemp, "meta");
+ assertTrue(metaDir.mkdir());
+
+ String tempDirValue = tempDir.getCanonicalPath();
+ String metaDirValue = metaDir.getCanonicalPath();
+
+ registry.add("tempDir", () -> tempDirValue);
+ registry.add("metaDir", () -> metaDirValue);
+ }
+
+ protected String sendNormalMessage(String msgBody) {
+ MessageProperties props = new MessageProperties();
+ props.setMessageId(UUID.randomUUID().toString());
+ props.setPriority(NORMAL_PRIORITY);
+ Message msg = new Message(msgBody.getBytes(StandardCharsets.UTF_8), props);
+ template.send(workerQueue.getActualName(), msg);
+ return props.getMessageId();
+ }
+
+ @BeforeEach
+ @SneakyThrows
+ void setup() {
+ checkChunkingProps(this.chunkingEnabled, this.chunkingByteSize);
+ purgeQueues();
+ setupKeystore();
+ setupDirectoriesAndFiles();
+ setupSFTP();
+ }
+
+ final void checkChunkingProps(boolean chunkingEnabled, String chunkingByteSize) {
+ assertTrue(chunkingEnabled);
+ assertEquals("20MB", chunkingByteSize);
+ }
+
+ @SneakyThrows
+ private void setupDirectoriesAndFiles() {
+ Path baseTemp = Files.createTempDirectory("tmpSftp");
+
+ sourceDir = baseTemp.resolve("source").toFile();
+ assertTrue(sourceDir.mkdir());
+ destDir = baseTemp.resolve("dest").toFile();
+ assertTrue(destDir.mkdir());
+ retrieveBaseDir = baseTemp.resolve("retrieve").toFile();
+ assertTrue(retrieveBaseDir.mkdir());
+ retrieveDir = retrieveBaseDir.toPath().resolve("ret-folder").toFile();
+ assertTrue(retrieveDir.mkdir());
+
+ log.info("source dir [{}]", sourceDir);
+ log.info("dest dir [{}]", destDir);
+ log.info("retrieve base dir [{}]", retrieveBaseDir);
+ log.info("retrieve dir [{}]", retrieveDir);
+
+ File sourceFileDir = new File(sourceDir, "src-path-1");
+ assertTrue(sourceFileDir.mkdir());
+ assertTrue(sourceFileDir.exists() && sourceFileDir.isDirectory());
+
+ File sourceFile = new File(sourceFileDir, "src-file-1");
+ Files.copy(this.largeFile.getFile().toPath(), sourceFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+
+ assertTrue(sourceFile.exists() && sourceFile.isFile());
+ assertEquals(50_000_000, sourceFile.length());
+ }
+
+ @SneakyThrows
+ void setupKeystore() {
+ Path baseTemp = Files.createTempDirectory("tmpKeyStore");
+ Encryption.addBouncyCastleSecurityProvider();
+ keyStorePath = baseTemp.resolve("test.ks").toFile().getCanonicalPath();
+ log.info("BASE TEMP IS AT [{}]", baseTemp.toFile().getCanonicalPath());
+ log.info("TEMP KEY IS AT [{}]", keyStorePath);
+ Encryption enc = new Encryption();
+ enc.setVaultEnable(false);
+ enc.setVaultPrivateKeyEncryptionKeyName(KEY_NAME_FOR_SSH);
+ enc.setVaultDataEncryptionKeyName(KEY_NAME_FOR_DATA);
+
+ enc.setKeystoreEnable(true);
+ enc.setKeystorePath(keyStorePath);
+ enc.setKeystorePassword(KEY_STORE_PASSWORD);
+
+ SecretKey keyForSSH = Encryption.generateSecretKey();
+ SecretKey keyForData = Encryption.generateSecretKey();
+
+ assertFalse(new File(keyStorePath).exists());
+
+ Encryption.saveSecretKeyToKeyStore(Encryption.getVaultPrivateKeyEncryptionKeyName(),
+ keyForSSH);
+ Encryption.saveSecretKeyToKeyStore(Encryption.getVaultDataEncryptionKeyName(),
+ keyForData);
+ assertTrue(new File(keyStorePath).exists());
+ }
+
+ void purgeQueues() {
+ rabbitAdmin.purgeQueue(workerQueue.getActualName(), false);
+ rabbitAdmin.purgeQueue(workerQueue.getActualName(), false);
+ assertEquals(0, rabbitAdmin.getQueueInfo(workerQueue.getActualName()).getMessageCount());
+ assertEquals(0, rabbitAdmin.getQueueInfo(workerQueue.getActualName()).getMessageCount());
+ }
+
+ @Test
+ @SneakyThrows
+ void testDepositThenRetrieve() {
+ assertEquals(0, destDir.listFiles().length);
+ String depositMessage = getSampleDepositMessage();
+ Deposit deposit = new ObjectMapper().readValue(depositMessage, Deposit.class);
+ //log.info("depositMessage {}", depositMessage);
+ sendNormalMessage(depositMessage);
+ waitUntil(this::foundComplete);
+
+ DepositEvents depositEvents = new DepositEvents(deposit, this.events);
+
+ checkDepositWorkedOkay(depositMessage, depositEvents);
+
+ buildAndSendRetrieveMessage(depositEvents);
+ checkRetrieve();
+
+ }
+
+ void waitUntil(Callable test) {
+ Awaitility.await().atMost(5, TimeUnit.MINUTES)
+ .pollInterval(Duration.ofSeconds(15))
+ .until(test);
+ }
+
+ @SneakyThrows
+ private void checkRetrieve() {
+
+ waitUntil(this::foundRetrieveComplete);
+
+ Container.ExecResult execResult = userDataSourceContainer.execInContainer("/tmp/findSrcFile1.sh");
+ assertEquals(0, execResult.getExitCode(), "cannot find 50MB file in container - exit code");
+
+ String filesInContainerStdOut = execResult.getStdout();
+ assertTrue(filesInContainerStdOut.contains("50000000"), "cannot find 50MB file in container - file size");
+
+ // we tar up the retrieved files on the container and copy them back to the local file system.
+ // we tried using a shared directory via a bind mount but had problems with it in CI/CD pipeline which uses DockerInDocker
+ Container.ExecResult tarResult = userDataSourceContainer.execInContainer("tar","cvf","retFolder.tar","-C","/tmp/retrieve", "ret-folder");
+ assertEquals(0, tarResult.getExitCode());
+
+ File destination = new File(retrieveDir.toString());
+ assertTrue(destination.isDirectory());
+ assertTrue(destination.exists());
+
+ File localTarFile = new File(retrieveBaseDir.toString(), "retFolder.tar");
+ userDataSourceContainer.copyFileFromContainer("/retFolder.tar", localTarFile.toString());
+ unTar(localTarFile);
+
+ log.info("FIN {}", retrieveDir.getCanonicalPath());
+ File[] dvTimestampDirs = retrieveDir.listFiles(file -> file.isDirectory() && file.getName().startsWith("dv_"));
+ File dvLatestTimestampDir = Arrays.stream(dvTimestampDirs).sorted(Comparator.comparing((File::lastModified)).reversed()).findFirst().get();
+ File retrieved = new File(dvLatestTimestampDir + "/src-path-1/src-file-1");
+
+ String digestOriginal = Verify.getDigest(this.largeFile.getFile());
+ String digestRetrieved = Verify.getDigest(retrieved);
+
+ assertEquals(digestOriginal, digestRetrieved);
+ }
+
+ private void unTar(File tarFile) {
+ unTar(tarFile.toPath(), tarFile.toPath().getParent());
+ }
+
+ @SneakyThrows
+ public static void unTar( Path pathInput, Path pathOutput ) {
+ TarArchiveInputStream tararchiveinputstream =
+ new TarArchiveInputStream(new BufferedInputStream(Files.newInputStream(pathInput)));
+
+ ArchiveEntry archiveEntry;
+ while( (archiveEntry = tararchiveinputstream.getNextEntry()) != null ) {
+ Path pathEntryOutput = pathOutput.resolve( archiveEntry.getName() );
+ if( archiveEntry.isDirectory() ) {
+ if( !Files.exists( pathEntryOutput ) )
+ Files.createDirectory( pathEntryOutput );
+ }
+ else
+ Files.copy( tararchiveinputstream, pathEntryOutput );
+ }
+
+ tararchiveinputstream.close();
+ }
+
+ boolean foundRetrieveComplete() {
+ return events.stream()
+ .anyMatch(e -> e.getClass().equals(RetrieveComplete.class));
+ }
+
+ boolean foundComplete() {
+ return events.stream()
+ .anyMatch(e -> e.getClass().equals(Complete.class));
+ }
+
+ Optional getExpectedNumberChunks(){
+ return Optional.of(3);
+ }
+
+ @SneakyThrows
+ private void checkDepositWorkedOkay(String depositMessage, DepositEvents depositEvents) {
+ Deposit deposit = mapper.readValue(depositMessage, Deposit.class);
+ String bagId = deposit.getProperties().get("bagId");
+
+ log.info("BROKER MSG COUNT {}", events.size());
+ File[] destFiles = destDir.listFiles();
+
+ int expectedNumFiles = getExpectedNumberChunks().isPresent() ? getExpectedNumberChunks().get() : 1;
+ assertEquals(expectedNumFiles, destFiles.length);
+
+ Arrays.sort(destFiles, Comparator.comparing(File::getName));
+
+ final Map chunkNumToEncChunk = new HashMap<>();
+
+ final ComputedEncryption computedEncryption = depositEvents.getComputedEncryption();
+ AESMode aesMode = AESMode.valueOf(computedEncryption.getAesMode());
+ assertEquals(AESMode.GCM, aesMode);
+ final ComputedDigest computedDigest = depositEvents.getComputedDigest();
+
+ final File decryptedTarFile;
+
+ if (getExpectedNumberChunks().isPresent()) {
+
+ int expectedNumberChunks = getExpectedNumberChunks().get();
+
+ for (int chunkNum = 1; chunkNum <= expectedNumberChunks; chunkNum++) {
+ File expectedEncChunk = destDir.toPath().resolve(bagId + ".tar." + chunkNum).toFile();
+ assertEquals(expectedEncChunk, destFiles[chunkNum - 1]);
+ chunkNumToEncChunk.put(chunkNum, expectedEncChunk);
+ }
+
+ for (int chunkNum = 1; chunkNum <= expectedNumberChunks; chunkNum++) {
+ File expectedEncChunk = chunkNumToEncChunk.get(chunkNum);
+ String encHash = computedEncryption.getEncChunkDigests().get(chunkNum);
+ assertEquals(encHash, getSha1Hash(expectedEncChunk));
+ }
+
+ Map chunkNumToDecryptedChunk = new HashMap<>();
+ for (int chunkNum = 1; chunkNum <= expectedNumberChunks; chunkNum++) {
+ File expectedEncChunk = chunkNumToEncChunk.get(chunkNum);
+ byte[] iv = computedEncryption.getChunkIVs().get(chunkNum);
+ File decryptedChunkFile = Files.createTempFile("decryptedChunk", ".plain").toFile();
+ chunkNumToDecryptedChunk.put(chunkNum, decryptedChunkFile);
+ FileUtils.copyFile(expectedEncChunk, decryptedChunkFile);
+ Encryption.decryptFile(aesMode, decryptedChunkFile, iv);
+ assertTrue(decryptedChunkFile.length() > 0);
+ assertTrue(decryptedChunkFile.length() != expectedEncChunk.length());
+ }
+
+ decryptedTarFile = Files.createTempFile("decryptedTar", ".plain").toFile();
+
+ try (FileOutputStream fos = new FileOutputStream(decryptedTarFile)) {
+ for (int chunkNum = 1; chunkNum <= expectedNumberChunks; chunkNum++) {
+ File decryptedChunkFile = chunkNumToDecryptedChunk.get(chunkNum);
+ Files.copy(decryptedChunkFile.toPath(), fos);
+ }
+ }
+
+ } else {
+ File expectedEncTar = destDir.toPath().resolve(bagId + ".tar").toFile();
+ assertEquals(expectedEncTar, destFiles[0]);
+
+ String encTarHash = computedEncryption.getEncTarDigest();
+ assertEquals(encTarHash, getSha1Hash(expectedEncTar));
+
+ decryptedTarFile = Files.createTempFile("decryptedTar", ".plain").toFile();
+
+ byte[] iv = computedEncryption.getTarIV();
+ FileUtils.copyFile(destFiles[0], decryptedTarFile);
+ Encryption.decryptFile(aesMode, decryptedTarFile, iv);
+ assertTrue(decryptedTarFile.length() > 0);
+ assertTrue(decryptedTarFile.length() != expectedEncTar.length());
+ }
+
+ Set tarEntryPaths = getPathsWithinTarFile(decryptedTarFile);
+
+ Path base = Paths.get(bagId);
+ assertThat(tarEntryPaths).containsExactlyInAnyOrder(
+ base.resolve("bagit.txt"),
+ base.resolve("manifest-md5.txt"),
+ base.resolve("tagmanifest-md5.txt"),
+
+ base.resolve("data/src-path-1/src-file-1"),
+
+ base.resolve("metadata/filetype.json"),
+ base.resolve("metadata/vault.json"),
+ base.resolve("metadata/external.txt"),
+ base.resolve("metadata/deposit.json"));
+
+ assertEquals(computedDigest.getDigest(), getSha1Hash(decryptedTarFile));
+ }
+
+ @SneakyThrows
+ private void buildAndSendRetrieveMessage(DepositEvents depositEvents) {
+ String retrieveMessage2 = depositEvents.generateRetrieveMessage(this.retrieveBaseDir, this.retrieveDir.getName());
+
+ ObjectNode childNode = mapper.convertValue(this.sftpTargetProps, ObjectNode.class);
+ JsonNode parentNode = mapper.readTree(retrieveMessage2);
+
+ ObjectNode locatedNode1 = (ObjectNode)parentNode.path("userFileStoreProperties");
+ locatedNode1.put("FILE-STORE-SRC-ID", childNode);
+
+ ObjectNode locatedNode2 = (ObjectNode)parentNode.path("userFileStoreClasses");
+ locatedNode2.put("FILE-STORE-SRC-ID", new TextNode(SFTPFileSystem.class.getName()));
+
+ String sftpRetrieveMessage = mapper.writer(SerializationFeature.INDENT_OUTPUT).writeValueAsString(parentNode);
+
+ sendNormalMessage(sftpRetrieveMessage);
+ }
+
+ @SneakyThrows
+ private String getSampleDepositMessage() {
+ String temp1 = FileUtils.readFileToString(this.depositMessage.getFile(),
+ StandardCharsets.UTF_8);
+ String temp2 = temp1.replaceAll("/tmp/dv/src", sourceDir.getCanonicalPath());
+ String localFSresult = temp2.replaceAll("/tmp/dv/dest", destDir.getCanonicalPath());
+
+ ObjectNode childNode = mapper.convertValue(this.sftpSrcProps, ObjectNode.class);
+ JsonNode parentNode = mapper.readTree(localFSresult);
+
+ ObjectNode locatedNode1 = (ObjectNode)parentNode.path("userFileStoreProperties");
+ locatedNode1.put("FILE-STORE-SRC-ID", childNode);
+
+ ObjectNode locatedNode2 = (ObjectNode)parentNode.path("userFileStoreClasses");
+ locatedNode2.put("FILE-STORE-SRC-ID", new TextNode(SFTPFileSystem.class.getName()));
+
+ String sftpFSresult = mapper.writer(SerializationFeature.INDENT_OUTPUT).writeValueAsString(parentNode);
+
+ return sftpFSresult;
+ }
+
+ @RabbitListener(queues = BaseQueueConfig.BROKER_QUEUE_NAME)
+ @SneakyThrows
+ void receiveBrokerMessage(Message message, Channel channel,
+ @Header(AmqpHeaders.DELIVERY_TAG) long deliveryTag) {
+ channel.basicAck(deliveryTag, false);
+
+ String msgBody = new String(message.getBody(), StandardCharsets.UTF_8);
+ events.add(extractEvent(msgBody));
+
+ log.info("Received message for broker [{}]", events.size());
+ }
+
+ @SneakyThrows
+ private Event extractEvent(String message) {
+ Event event = mapper.readValue(message, Event.class);
+ String eventClassName = event.getEventClass();
+ Class eventClass = (Class) Class.forName(eventClassName);
+ return mapper.readValue(message, eventClass);
+ }
+
+ @SneakyThrows
+ private String getSha1Hash(File file) {
+ return Verify.getDigest(file);
+ }
+
+ public DockerImageName getDockerImageForOpenSSH() {
+ return DockerImageName.parse(DockerImage.OPEN_SSH_8pt6_IMAGE_NAME);
+ }
+
+
+ @SneakyThrows
+ public void setupSFTP() {
+
+ UserKeyPairService keyPairService = new UserKeyPairServiceImpl(TEST_PASSPHRASE);
+
+ UserKeyPairService.KeyPairInfo keypairInfo = keyPairService.generateNewKeyPair();
+ this.sftpPrivateKey = keypairInfo.getPrivateKey();
+ this.sftpPublicKey = keypairInfo.getPublicKey();
+
+ userDataSourceContainer = new GenericContainer<>(getDockerImageForOpenSSH())
+ .withEnv(ENV_USER_NAME, TEST_USER)
+ .withEnv(ENV_PUBLIC_KEY, sftpPublicKey) //this causes the public key to be added to /config/.ssh/authorized_keys
+ .withExposedPorts(2222)
+ //.withFileSystemBind(this.sourceDir.getCanonicalPath(), "/tmp/source")
+ //.withFileSystemBind(this.retrieveDir.getCanonicalPath(), "/tmp/retrieve/ret-folder")
+ .withCopyFileToContainer(MountableFile.forHostPath(sourceDir.toPath()),"/tmp/source")
+ .withCopyFileToContainer(MountableFile.forClasspathResource("docker/findSrcFile1.sh"),"/tmp/findSrcFile1.sh")
+ .waitingFor(Wait.forListeningPort());
+
+ userDataSourceContainer.start();
+
+ Container.ExecResult mkdirResult = userDataSourceContainer.execInContainer("mkdir", "-p", "/tmp/retrieve/ret-folder");
+ log.info("mkdir exit code [{}]", mkdirResult.getExitCode());
+
+ Container.ExecResult execresult1 = userDataSourceContainer.execInContainer("chown", "-R", "testuser", "/tmp/retrieve");
+ log.info("chown exit code [{}]", execresult1.getExitCode());
+
+ Container.ExecResult execresult2 = userDataSourceContainer.execInContainer("chmod", "a+r", "/tmp/retrieve");
+ log.info("chmod exit code [{}]", execresult2.getExitCode());
+
+ Container.ExecResult execresult3 = userDataSourceContainer.execInContainer("chmod", "+x", "/tmp/findSrcFile1.sh");
+ log.info("chmod exit code [{}]", execresult3.getExitCode());
+
+ byte[] iv = Encryption.generateIV();
+
+ byte[] encSftpPrivateKey = Encryption.encryptSecret(sftpPrivateKey, iv);
+
+ sftpSrcProps = new HashMap<>();
+ sftpSrcProps.put(PropNames.PRIVATE_KEY, Base64.getEncoder().encodeToString(encSftpPrivateKey));
+ sftpSrcProps.put(PropNames.IV, Base64.getEncoder().encodeToString(iv));
+ sftpSrcProps.put(PropNames.PASSPHRASE, TEST_PASSPHRASE);
+ sftpSrcProps.put(PropNames.USERNAME, TEST_USER);
+ sftpSrcProps.put(PropNames.HOST, this.userDataSourceContainer.getHost());
+ sftpSrcProps.put(PropNames.PORT, ""+this.userDataSourceContainer.getMappedPort(2222));
+ sftpSrcProps.put(PropNames.ROOT_PATH, "/tmp/source");
+
+ sftpTargetProps = new HashMap<>(sftpSrcProps);
+ sftpTargetProps.put(PropNames.ROOT_PATH, "/tmp/retrieve");
+
+ assertNotEquals(sftpTargetProps.get(PropNames.ROOT_PATH), sftpSrcProps.get(PropNames.ROOT_PATH));
+ }
+}
\ No newline at end of file
diff --git a/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/PerformDepositThenRetrieveUsingSftpWithApacheSshdIT.java b/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/PerformDepositThenRetrieveUsingSftpWithApacheSshdIT.java
new file mode 100644
index 000000000..b08ca4a66
--- /dev/null
+++ b/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/PerformDepositThenRetrieveUsingSftpWithApacheSshdIT.java
@@ -0,0 +1,18 @@
+package org.datavaultplatform.worker.tasks.sftp;
+
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.worker.app.DataVaultWorkerInstanceApp;
+import org.datavaultplatform.worker.test.AddTestProperties;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.TestPropertySource;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+@SpringBootTest(classes = DataVaultWorkerInstanceApp.class)
+@Testcontainers(disabledWithoutDocker = true)
+@AddTestProperties
+@DirtiesContext
+@TestPropertySource(properties = {"sftp.driver.use.apache.sshd=false"})
+@Slf4j
+public class PerformDepositThenRetrieveUsingSftpWithApacheSshdIT extends BasePerformDepositThenRetrieveUsingSftpIT {
+}
\ No newline at end of file
diff --git a/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/PerformDepositThenRetrieveUsingSftpWithJschIT.java b/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/PerformDepositThenRetrieveUsingSftpWithJschIT.java
new file mode 100644
index 000000000..d9c07988b
--- /dev/null
+++ b/datavault-worker/src/test/java/org/datavaultplatform/worker/tasks/sftp/PerformDepositThenRetrieveUsingSftpWithJschIT.java
@@ -0,0 +1,18 @@
+package org.datavaultplatform.worker.tasks.sftp;
+
+import lombok.extern.slf4j.Slf4j;
+import org.datavaultplatform.worker.app.DataVaultWorkerInstanceApp;
+import org.datavaultplatform.worker.test.AddTestProperties;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.TestPropertySource;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+@SpringBootTest(classes = DataVaultWorkerInstanceApp.class)
+@Testcontainers(disabledWithoutDocker = true)
+@AddTestProperties
+@DirtiesContext
+@TestPropertySource(properties = {"sftp.driver.use.apache.sshd=true"})
+@Slf4j
+public class PerformDepositThenRetrieveUsingSftpWithJschIT extends BasePerformDepositThenRetrieveUsingSftpIT {
+}
\ No newline at end of file
diff --git a/datavault-worker/src/test/resources/docker/findSrcFile1.sh b/datavault-worker/src/test/resources/docker/findSrcFile1.sh
new file mode 100644
index 000000000..29a9eac53
--- /dev/null
+++ b/datavault-worker/src/test/resources/docker/findSrcFile1.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+# see BasePerformDepositThenRetrieveUsingSftpIT.java
+find /tmp/retrieve -type f -name src-file-1 -exec ls -l {} \;
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 3ff80c88c..1a3cf8816 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,7 +12,7 @@
org.springframework.boot
spring-boot-starter-parent
- 2.7.6
+ 2.7.9
@@ -35,23 +35,37 @@
false
+
+
+ 3.1.2
+
- 1.12.334
+ 1.12.426
- 3.0.1
-
-
- 1.17.6
+ 3.7.0
- 3.1.5
+ 3.1.6
- 4.9.0
+ 4.11.0
- 7.3.1
+ 8.1.2
+
+
+ 1.15.4
+
+
+
+
+ 7.5
+
+
+
+
+ 1.17.6
1.22
@@ -73,9 +87,6 @@
1.10.0
-
- 1.15.3
-
5.1.0
@@ -94,9 +105,6 @@
5.2.0
-
- 6.14.3
-
2.3
@@ -191,13 +199,6 @@
pom
import
-
- com.oracle.oci.sdk
- oci-java-sdk-bom
- ${oci.java.sdk.bom.version}
- pom
- import
-
com.oracle.oci.sdk
oci-java-sdk-common-httpclient-jersey