Feature: Stop solidifying when asked for pruned transaction

src/main/java/com/iota/iri/Iota.java

@@ -21,8 +21,12 @@
 import com.iota.iri.service.spentaddresses.impl.SpentAddressesServiceImpl;
 import com.iota.iri.service.tipselection.*;
 import com.iota.iri.service.tipselection.impl.*;
+import com.iota.iri.service.transactionpruning.PrunedTransactionException;
+import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier;
 import com.iota.iri.service.transactionpruning.TransactionPruningException;
 import com.iota.iri.service.transactionpruning.async.AsyncTransactionPruner;
+import com.iota.iri.service.transactionpruning.impl.PrunedTransactionProviderImpl;
+import com.iota.iri.service.transactionpruning.impl.PrunedTransactionVerifierImpl;
 import com.iota.iri.storage.*;
 import com.iota.iri.storage.rocksDB.RocksDBPersistenceProvider;
 import com.iota.iri.utils.Pair;
@@ -95,6 +99,10 @@ public class Iota {
 
     public final TransactionRequesterWorkerImpl transactionRequesterWorker;
 
+    public final PrunedTransactionProviderImpl prunedTransactionProvider;
+
+    public final PrunedTransactionVerifier prunedTransactionVerifier;
+
     public final BundleValidator bundleValidator;
 
     public final Tangle tangle;
@@ -108,15 +116,18 @@ public class Iota {
     public final TipsViewModel tipsViewModel;
     public final TipSelector tipsSelector;
 
+
+
     /**
      * Initializes the latest snapshot and then creates all services needed to run an IOTA node.
      *
      * @param configuration Information about how this node will be configured.
      * @throws TransactionPruningException If the TransactionPruner could not restore its state.
      * @throws SnapshotException If the Snapshot fails to initialize.
      *                           This can happen if the snapshot signature is invalid or the file cannot be read.
+     * @throws PrunedTransactionException If we could not load previously pruned transactions (whilst they exist)
      */
-    public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException {
+    public Iota(IotaConfig configuration) throws TransactionPruningException, SnapshotException, SpentAddressesException, PrunedTransactionException {
         this.configuration = configuration;
 
         // new refactored instances
@@ -135,6 +146,7 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh
         transactionPruner = configuration.getLocalSnapshotsEnabled() && configuration.getLocalSnapshotsPruningEnabled()
                           ? new AsyncTransactionPruner()
                           : null;
+
         transactionRequesterWorker = new TransactionRequesterWorkerImpl();
 
         // legacy code
@@ -143,8 +155,13 @@ public Iota(IotaConfig configuration) throws TransactionPruningException, Snapsh
         tipsViewModel = new TipsViewModel();
         transactionRequester = new TransactionRequester(tangle, snapshotProvider);
         transactionValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester);
+
+        prunedTransactionProvider = transactionPruner != null ? new PrunedTransactionProviderImpl() : null;
+        prunedTransactionVerifier = transactionPruner != null ? new PrunedTransactionVerifierImpl(
+                prunedTransactionProvider, transactionRequester) : null;
+
         node = new Node(tangle, snapshotProvider, transactionValidator, transactionRequester, tipsViewModel,
-                latestMilestoneTracker, configuration);
+                latestMilestoneTracker, configuration, prunedTransactionVerifier);
         replicator = new Replicator(node, configuration);
         udpReceiver = new UDPReceiver(node, configuration);
         tipsSolidifier = new TipsSolidifier(tangle, transactionValidator, tipsViewModel, configuration);
@@ -175,7 +192,7 @@ public void init() throws Exception {
             tangle.clearMetadata(com.iota.iri.model.persistables.Transaction.class);
         }
 
-        transactionValidator.init(configuration.isTestnet(), configuration.getMwm());
+        transactionValidator.init(configuration.isTestnet(), configuration.getMwm(), prunedTransactionVerifier);
         tipsSolidifier.init();
         transactionRequester.init(configuration.getpRemoveRequest());
         udpReceiver.init();
@@ -196,7 +213,9 @@ public void init() throws Exception {
         }
     }
 
-    private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException {
+    private void injectDependencies() throws SnapshotException, TransactionPruningException, SpentAddressesException, 
+    PrunedTransactionException {
+
         //snapshot provider must be initialized first
         //because we check whether spent addresses data exists
         snapshotProvider.init(configuration);
@@ -216,8 +235,14 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx
         ledgerService.init(tangle, snapshotProvider, snapshotService, milestoneService, spentAddressesService,
                 bundleValidator);
         if (transactionPruner != null) {
-            transactionPruner.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, tipsViewModel, configuration);
+            if (prunedTransactionProvider != null) {
+                prunedTransactionProvider.init(configuration);
+            }
+            transactionPruner.init(tangle, snapshotProvider, spentAddressesService, spentAddressesProvider, 
+                    tipsViewModel, configuration, prunedTransactionProvider);
         }
+
+
         transactionRequesterWorker.init(tangle, transactionRequester, tipsViewModel, node);
     }
 

src/main/java/com/iota/iri/TransactionValidator.java

@@ -10,6 +10,7 @@
 import com.iota.iri.model.TransactionHash;
 import com.iota.iri.network.TransactionRequester;
 import com.iota.iri.service.snapshot.SnapshotProvider;
+import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier;
 import com.iota.iri.storage.Tangle;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -52,6 +53,8 @@ public class TransactionValidator {
     private final Set<Hash> newSolidTransactionsOne = new LinkedHashSet<>();
     private final Set<Hash> newSolidTransactionsTwo = new LinkedHashSet<>();
 
+    private PrunedTransactionVerifier prunedTransactionVerifier;
+
     /**
      * Constructor for Tangle Validator
      *
@@ -81,9 +84,12 @@ public class TransactionValidator {
      *                regardless of parameter input.
      * @param mwm minimum weight magnitude: the minimal number of 9s that ought to appear at the end of the transaction
      *            hash
+     * @param prunedTransactionProviderProvider for checking if a transaction is pruned previously
      */
-    public void init(boolean testnet, int mwm) {
+    public void init(boolean testnet, int mwm, PrunedTransactionVerifier prunedTransactionVerifier) {
         setMwm(testnet, mwm);
+
+        this.prunedTransactionVerifier = prunedTransactionVerifier;
 
         newSolidThread = new Thread(spawnSolidTransactionsPropagation(), "Solid TX cascader");
         newSolidThread.start();
@@ -247,6 +253,13 @@ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTrans
         if(fromHash(tangle, hash).isSolid()) {
             return true;
         }
+
+        // isPruned gets updated through parent CF verification. 
+        // If one check is negative, isPossiblyPruned will return false.
+        if (this.prunedTransactionVerifier != null && this.prunedTransactionVerifier.isPossiblyPruned(hash)){
+            return this.prunedTransactionVerifier.isPruned(hash);
+        }
+
         Set<Hash> analyzedHashes = new HashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet());
         if(maxProcessedTransactions != Integer.MAX_VALUE) {
             maxProcessedTransactions += analyzedHashes.size();

src/main/java/com/iota/iri/conf/BaseIotaConfig.java

@@ -5,7 +5,6 @@
 import com.beust.jcommander.ParameterException;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.iota.iri.IRI;
 import com.iota.iri.crypto.SpongeFactory;
 import com.iota.iri.model.Hash;
 import com.iota.iri.model.HashFactory;
@@ -111,7 +110,9 @@ public abstract class BaseIotaConfig implements IotaConfig {
     protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH;
     protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH;
     protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH;
-
+    protected String prunedTransactionsDbPath = Defaults.PRUNED_TRANSACTIONS_DB_PATH;
+    protected String prunedTransactionsDbLogPath = Defaults.PRUNED_TRANSACTIONS_DB_LOG_PATH;
+
     public BaseIotaConfig() {
         //empty constructor
     }
@@ -667,13 +668,35 @@ protected void setSpentAddressesDbPath(String spentAddressesDbPath) {
     public String getSpentAddressesDbLogPath() {
         return spentAddressesDbLogPath;
     }
-
+    
     @JsonProperty
     @Parameter(names = {"--spent-addresses-db-log-path"}, description = SnapshotConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH)
     protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) {
         this.spentAddressesDbLogPath = spentAddressesDbLogPath;
     }
-
+
+    @Override
+    public String getPrunedTransactionsDbLogPath() {
+        return prunedTransactionsDbLogPath;
+    }
+
+    @JsonProperty
+    @Parameter(names = {"--pruned-transactions-db-log-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_LOG_PATH)
+    protected void setPrunedTransactionDbLogPath(String prunedTransactionsDbLogPath) {
+        this.prunedTransactionsDbLogPath = prunedTransactionsDbLogPath;
+    }
+
+    @Override
+    public String getPrunedTransactionsDbPath() {
+        return prunedTransactionsDbPath;
+    }
+
+    @JsonProperty
+    @Parameter(names = {"--pruned-transactions-db-path"}, description = SnapshotConfig.Descriptions.PRUNED_TRANSACTIONS_DB_PATH)
+    protected void setPrunedTransactionsDbPath(String prunedTransactionsDbPath) {
+        this.prunedTransactionsDbPath = prunedTransactionsDbPath;
+    }
+
     /**
      * Checks if ZMQ is enabled.
      * @return true if zmqEnableTcp or zmqEnableIpc is set.
@@ -951,6 +974,9 @@ public interface Defaults {
         int LOCAL_SNAPSHOTS_DEPTH_MIN = 100;
         String SPENT_ADDRESSES_DB_PATH = "spent-addresses-db";
         String SPENT_ADDRESSES_DB_LOG_PATH = "spent-addresses-log";
+
+        String PRUNED_TRANSACTIONS_DB_LOG_PATH = "spent-addresses-db";
+        String PRUNED_TRANSACTIONS_DB_PATH = "spent-addresses-log";
 
         String LOCAL_SNAPSHOTS_BASE_PATH = "mainnet";
         String SNAPSHOT_FILE = "/snapshotMainnet.txt";

src/main/java/com/iota/iri/conf/SnapshotConfig.java

@@ -76,6 +76,16 @@ public interface SnapshotConfig extends Config {
      */
     String getSpentAddressesDbLogPath();
 
+    /**
+     * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_PATH}
+     */
+    String getPrunedTransactionsDbPath();
+
+    /**
+     * @return {@value Descriptions#PRUNED_TRANSACTIONS_DB_LOG_PATH}
+     */
+    String getPrunedTransactionsDbLogPath();
+
     interface Descriptions {
 
         String LOCAL_SNAPSHOTS_ENABLED = "Flag that determines if local snapshots are enabled.";
@@ -94,5 +104,8 @@ interface Descriptions {
                 "from previous epochs";
         String SPENT_ADDRESSES_DB_PATH = "The folder where the spent addresses DB saves its data.";
         String SPENT_ADDRESSES_DB_LOG_PATH = "The folder where the spent addresses DB saves its logs.";
+        String PRUNED_TRANSACTIONS_DB_PATH = "The folder where the pruned transactions DB saves its data.";
+        String PRUNED_TRANSACTIONS_DB_LOG_PATH = "The folder where the pruned transactions DB saves its logs.";
     }
+
 }

src/main/java/com/iota/iri/model/persistables/Cuckoo.java

@@ -0,0 +1,82 @@
+package com.iota.iri.model.persistables;
+
+import java.util.BitSet;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import com.iota.iri.model.IntegerIndex;
+import com.iota.iri.storage.Persistable;
+import com.iota.iri.utils.Serializer;
+
+/**
+ * Persistable to manage the data we get as a result of pruning a milestone and its transactions
+ */
+public class Cuckoo implements Persistable {
+
+    /**
+     * The filter number it belonged to in previous cycles
+     */
+    public IntegerIndex filterId;
+
+    /**
+     * The bits that make up the CF
+     */
+    public BitSet filterBits;
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public byte[] bytes() {
+        byte[] num = filterId.bytes();
+        return ArrayUtils.addAll(num, filterBits.toByteArray());
+    }
+
+    /**
+     * Reads a CuckooBucket from the provided bytes.
+     * First 4 bytes are the bucket id, second 4 are the index inside that bucket
+     * Rest of the bytes is the bucket data
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public void read(byte[] bytes) {
+        if(bytes != null) {
+            filterId = new IntegerIndex(Serializer.getInteger(bytes, 0));
+
+            short start = 4;
+            filterBits = new BitSet(bytes.length - start);
+            for (int i = start; i < bytes.length; i++) {
+                filterBits.set(i-start, bytes[i]);
+            }
+        }
+    }
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public byte[] metadata() {
+        return new byte[0];
+    }
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public void readMetadata(byte[] bytes) {
+    }
+
+    /**
+     * 
+     * {@inheritDoc}
+     */
+    @Override
+    public boolean merge() {
+        return false;
+    }
+
+}

src/main/java/com/iota/iri/network/Node.java

@@ -10,6 +10,8 @@
 import com.iota.iri.model.TransactionHash;
 import com.iota.iri.service.milestone.LatestMilestoneTracker;
 import com.iota.iri.service.snapshot.SnapshotProvider;
+import com.iota.iri.service.transactionpruning.PrunedTransactionException;
+import com.iota.iri.service.transactionpruning.PrunedTransactionVerifier;
 import com.iota.iri.storage.Tangle;
 import net.openhft.hashing.LongHashFunction;
 import org.apache.commons.lang3.StringUtils;
@@ -69,7 +71,8 @@ public class Node {
     private final TransactionValidator transactionValidator;
     private final LatestMilestoneTracker latestMilestoneTracker;
     private final TransactionRequester transactionRequester;
-
+    private final PrunedTransactionVerifier prunedTransactionVerifier;
+
     private static final SecureRandom rnd = new SecureRandom();
 
 
@@ -99,7 +102,10 @@ public class Node {
      * @param configuration Contains all the config.
      *
      */
-    public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration
+    public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, 
+            final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, 
+            final LatestMilestoneTracker latestMilestoneTracker, final NodeConfig configuration,
+            final PrunedTransactionVerifier prunedTransactionVerifier
     ) {
         this.configuration = configuration;
         this.tangle = tangle;
@@ -112,6 +118,7 @@ public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final Transa
         int packetSize = configuration.getTransactionPacketSize();
         this.sendingPacket = new DatagramPacket(new byte[packetSize], packetSize);
         this.tipRequestingPacket = new DatagramPacket(new byte[packetSize], packetSize);
+        this.prunedTransactionVerifier = prunedTransactionVerifier;
 
     }
 
@@ -445,6 +452,20 @@ public void replyToRequestFromQueue() {
     public void processReceivedData(TransactionViewModel receivedTransactionViewModel, Neighbor neighbor) {
 
         boolean stored = false;
+
+        try {
+            if (prunedTransactionVerifier != null && 
+                    prunedTransactionVerifier.waitingForHash(receivedTransactionViewModel.getHash())) {
+
+                prunedTransactionVerifier.submitTransaction(receivedTransactionViewModel);
+
+                // We dont store these old/pruned transactions
+                return;
+            }
+        } catch (PrunedTransactionException e) {
+            // We could not verify if this was pruned or not. Handle like normal
+            log.warn("Failed checking for pruned transaction state.", e);
+        }
 
         //store new transaction
         try {

src/main/java/com/iota/iri/service/spentaddresses/impl/SpentAddressesProviderImpl.java

@@ -54,8 +54,7 @@ public SpentAddressesProviderImpl init(SnapshotConfig config)
                     {{put("spent-addresses", SpentAddress.class);}}, null);
             this.rocksDBPersistenceProvider.init();
             readPreviousEpochsSpentAddresses();
-        }
-        catch (Exception e) {
+        } catch (Exception e) {
             throw new SpentAddressesException("There is a problem with accessing stored spent addresses", e);
         }
         return this;