diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json index 5b64c35a..059625f3 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 823635.7718335792 + "mean_ops" : 819668.8832917466 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json index fe461876..9a4d8c6d 100644 --- a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 655010.8023043653 + "mean_ops" : 664604.8239805239 } \ No newline at end of file diff --git a/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareNonceGeneratorTest.testGenerateWithBenchmark.json b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareNonceGeneratorTest.testGenerateWithBenchmark.json new file mode 100644 index 00000000..79ec5194 --- /dev/null +++ b/ranger-discovery-bundle/perf/results/io.appform.ranger.discovery.bundle.id.PartitionAwareNonceGeneratorTest.testGenerateWithBenchmark.json @@ -0,0 +1,7 @@ +{ + "name" : "io.appform.ranger.discovery.bundle.id.PartitionAwareNonceGeneratorTest.testGenerateWithBenchmark", + "iterations" : 100000, + "threads" : 5, + "totalMillis" : 8204, + "avgTime" : 1640.8 +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CircularQueue.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CircularQueue.java new file mode 100644 index 00000000..272c2907 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CircularQueue.java @@ -0,0 +1,69 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import lombok.val; + +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; + +class CircularQueue { + private static final String QUEUE_FULL_METRIC_STRING = "idGenerator.queueFull.forPrefix."; + private static final String UNUSED_INDICES_METRIC_STRING = "idGenerator.unusedIds.forPrefix."; + + private final Meter queueFullMeter; + private final Meter unusedDataMeter; + + /** List of data for the specific queue */ + private final AtomicIntegerArray queue; + + /** Size of the queue */ + private final int size; + + /** Pointer to track the first usable index. Helps to determine the next usable data. */ + private final AtomicInteger firstIdx = new AtomicInteger(0); + + /** Pointer to track index of last usable index. Helps to determine which index the next data should go in. */ + private final AtomicInteger lastIdx = new AtomicInteger(0); + + + public CircularQueue(int size, final MetricRegistry metricRegistry, final String namespace) { + this.size = size; + this.queue = new AtomicIntegerArray(size); + this.queueFullMeter = metricRegistry.meter(QUEUE_FULL_METRIC_STRING + namespace); + this.unusedDataMeter = metricRegistry.meter(UNUSED_INDICES_METRIC_STRING + namespace); + } + + public synchronized void setId(int id) { + // Don't store new data if the queue is already full of unused data. + if (lastIdx.get() >= firstIdx.get() + size - 1) { + queueFullMeter.mark(); + return; + } + val arrayIdx = lastIdx.get() % size; + queue.set(arrayIdx, id); + lastIdx.getAndIncrement(); + } + + private int getId(int index) { + val arrayIdx = index % size; + return queue.get(arrayIdx); + } + + public synchronized Optional getNextId() { + if (firstIdx.get() < lastIdx.get()) { + val id = getId(firstIdx.getAndIncrement()); + return Optional.of(id); + } else { + return Optional.empty(); + } + } + + public void reset() { + val unusedIds = lastIdx.get() - firstIdx.get(); + unusedDataMeter.mark(unusedIds); + lastIdx.set(0); + firstIdx.set(0); + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java index 0f44eb14..a94083ef 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/Constants.java @@ -24,4 +24,7 @@ public class Constants { public static final int MAX_ID_PER_MS = 1000; public static final int MAX_NUM_NODES = 10000; + public static final int MAX_IDS_PER_SECOND = 1_000_000; + public static final int DEFAULT_DATA_STORAGE_TIME_LIMIT_IN_SECONDS = 60; + public static final int DEFAULT_PARTITION_RETRY_COUNT = 100; } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java index e29e910b..0ee1082f 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/GenerationResult.java @@ -9,4 +9,5 @@ public class GenerationResult { NonceInfo nonceInfo; IdValidationState state; Domain domain; + String namespace; } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java index 4c4c29e6..bd45c15f 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdGenerator.java @@ -38,7 +38,7 @@ public class IdGenerator { private static final IdGeneratorBase baseGenerator = new DefaultIdGenerator(); public static void initialize(int node) { - baseGenerator.setNodeId(node); + baseGenerator.setNODE_ID(node); } public static synchronized void cleanUp() { diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java new file mode 100644 index 00000000..92e7f4a3 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdPool.java @@ -0,0 +1,26 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.codahale.metrics.MetricRegistry; + +import java.util.Optional; + +public class IdPool { + /** List of IDs for the specific IdPool */ + private final CircularQueue queue; + + public IdPool(int size, final MetricRegistry metricRegistry, final String namespace) { + this.queue = new CircularQueue(size, metricRegistry, namespace); + } + + public void setId(int id) { + queue.setId(id); + } + + public Optional getNextId() { + return queue.getNextId(); + } + + public void reset() { + queue.reset(); + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdUtils.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdUtils.java new file mode 100644 index 00000000..c4942364 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/IdUtils.java @@ -0,0 +1,25 @@ +package io.appform.ranger.discovery.bundle.id; + +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; +import io.appform.ranger.discovery.bundle.id.generator.IdGeneratorBase; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.joda.time.DateTime; + +@Slf4j +@UtilityClass +public class IdUtils { + + public Id getIdFromNonceInfo(final NonceInfo idInfo, final String namespace, final IdFormatter idFormatter) { + val dateTime = new DateTime(idInfo.getTime()); + val id = String.format("%s%s", namespace, idFormatter.format(dateTime, IdGeneratorBase.getNODE_ID(), idInfo.getExponent())); + return Id.builder() + .id(id) + .exponent(idInfo.getExponent()) + .generatedDate(dateTime.toDate()) + .node(IdGeneratorBase.getNODE_ID()) + .build(); + } + +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java new file mode 100644 index 00000000..b68c628e --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/PartitionIdTracker.java @@ -0,0 +1,76 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import com.google.common.base.Preconditions; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.time.Instant; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.appform.ranger.discovery.bundle.id.Constants.MAX_IDS_PER_SECOND; + +/** + * Tracks generated IDs and pointers for generating next IDs for a partition. + */ +@Slf4j +public class PartitionIdTracker { + /** Array to store IdPools for each partition */ + private final IdPool[] idPoolList; + + /** Counter to keep track of the number of IDs created */ + private final AtomicInteger nextIdCounter = new AtomicInteger(); + + @Getter + private Instant instant; + + private final Meter generatedIdCountMeter; + + public PartitionIdTracker(final int partitionSize, + final int idPoolSize, + final Instant instant, + final MetricRegistry metricRegistry, + final String namespace) { + this.instant = instant; + idPoolList = new IdPool[partitionSize]; + for (int i=0; i namespaceConfig = Collections.emptySet(); + + @NotNull + @Min(1) + private int partitionCount; + + /** Buffer time in seconds to pre-generate IDs for. + * This allows us to generate unique IDs even if the clock gets skewed. */ + @Min(1) + @Max(300) + @Builder.Default + private int dataStorageLimitInSeconds = Constants.DEFAULT_DATA_STORAGE_TIME_LIMIT_IN_SECONDS; + + @ValidationMethod(message = "Namespaces should be unique") + @JsonIgnore + public boolean areNamespacesUnique() { + Set namespaces = namespaceConfig.stream() + .map(NamespaceConfig::getNamespace) + .collect(Collectors.toSet()); + return namespaceConfig.size() == namespaces.size(); + } + +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/NamespaceConfig.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/NamespaceConfig.java new file mode 100644 index 00000000..f9e6255b --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/config/NamespaceConfig.java @@ -0,0 +1,22 @@ +package io.appform.ranger.discovery.bundle.id.config; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class NamespaceConfig { + @NotNull + private String namespace; + + /** Size of pre-generated id buffer. Value from DefaultNamespaceConfig will be used if this is null */ + @Min(2) + private Integer idPoolSizePerBucket; +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/Base36IdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/Base36IdFormatter.java index 15db24bb..e53aac78 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/Base36IdFormatter.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/Base36IdFormatter.java @@ -29,6 +29,11 @@ public Base36IdFormatter(IdFormatter idFormatter) { this.idFormatter = idFormatter; } + @Override + public IdParserType getType() { + throw new UnsupportedOperationException(); + } + @Override public String format(final DateTime dateTime, final int nodeId, diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DefaultIdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DefaultIdFormatter.java index cac2bbcb..3cb6ac51 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DefaultIdFormatter.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/DefaultIdFormatter.java @@ -28,6 +28,11 @@ public class DefaultIdFormatter implements IdFormatter { private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{15})([0-9]{4})([0-9]{3})"); private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmssSSS"); + @Override + public IdParserType getType() { + return IdParserType.DEFAULT; + } + @Override public String format(final DateTime dateTime, final int nodeId, diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatter.java index 9cd3219e..0fa33809 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatter.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatter.java @@ -22,6 +22,8 @@ public interface IdFormatter { + IdParserType getType(); + String format(final DateTime dateTime, final int nodeId, final int randomNonce); diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java index b0ea164f..8acc3dd0 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdFormatters.java @@ -22,6 +22,7 @@ public class IdFormatters { private static final IdFormatter originalIdFormatter = new DefaultIdFormatter(); private static final IdFormatter base36IdFormatter = new Base36IdFormatter(originalIdFormatter); + private static final IdFormatter secondPrecisionIdFormatter = new SecondPrecisionIdFormatter(); public static IdFormatter original() { return originalIdFormatter; @@ -31,4 +32,8 @@ public static IdFormatter base36() { return base36IdFormatter; } + public static IdFormatter secondPrecision() { + return secondPrecisionIdFormatter; + } + } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParserType.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParserType.java new file mode 100644 index 00000000..d7965bb1 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParserType.java @@ -0,0 +1,15 @@ +package io.appform.ranger.discovery.bundle.id.formatter; + +import lombok.Getter; + +@Getter +public enum IdParserType { + DEFAULT (0), + SECOND_PRECISION (10); + + private final int value; + + IdParserType(final int value) { + this.value = value; + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParsers.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParsers.java index 00b3520a..ef77fcce 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParsers.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/IdParsers.java @@ -20,14 +20,20 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; +import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; + @Slf4j @UtilityClass public class IdParsers { private static final int MINIMUM_ID_LENGTH = 22; - private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{22})"); + private static final Pattern PATTERN = Pattern.compile("([A-Za-z]*)([0-9]{22})([0-9]{2})?(.*)"); + + private final Map parserRegistry = Map.of( + IdFormatters.original().getType().getValue(), IdFormatters.original() + ); /** * Parse the given string to get ID @@ -44,7 +50,18 @@ public Optional parse(final String idString) { if (!matcher.find()) { return Optional.empty(); } - return IdFormatters.original().parse(idString); + + val parserType = matcher.group(3); + if (parserType == null) { + return IdFormatters.original().parse(idString); + } + + val parser = parserRegistry.get(Integer.parseInt(matcher.group(3))); + if (parser == null) { + log.warn("Could not parse idString {}, Invalid formatter type {}", idString, parserType); + return Optional.empty(); + } + return parser.parse(idString); } catch (Exception e) { log.warn("Could not parse idString {}", e.getMessage()); return Optional.empty(); diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/SecondPrecisionIdFormatter.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/SecondPrecisionIdFormatter.java new file mode 100644 index 00000000..679d61c7 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/formatter/SecondPrecisionIdFormatter.java @@ -0,0 +1,45 @@ +package io.appform.ranger.discovery.bundle.id.formatter; + +import io.appform.ranger.discovery.bundle.id.Id; +import lombok.val; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.util.Optional; +import java.util.regex.Pattern; + + +public class SecondPrecisionIdFormatter implements IdFormatter { + private static final Pattern PATTERN = Pattern.compile("(.*?)([0-9]{12})([0-9]{3})([0-9]{4})([0-9]{3})([0-9]{2}?)"); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); + + @Override + public IdParserType getType() { + return IdParserType.SECOND_PRECISION; + } + + @Override + public String format(final DateTime dateTime, + final int nodeId, + final int randomNonce) { + val leftRandom = randomNonce / 1000; + val rightRandom = randomNonce % 1000; + return String.format("%s%03d%04d%03d", DATE_TIME_FORMATTER.print(dateTime), leftRandom, nodeId, rightRandom); + } + + @Override + public Optional parse(String idString) { + val matcher = PATTERN.matcher(idString); + if (!matcher.find()) { + return Optional.empty(); + } + val exponent = (1000 * Integer.parseInt(matcher.group(3))) + Integer.parseInt(matcher.group(5)); + return Optional.of(Id.builder() + .id(idString) + .node(Integer.parseInt(matcher.group(4))) + .exponent(exponent) + .generatedDate(DATE_TIME_FORMATTER.parseDateTime(matcher.group(2)).toDate()) + .build()); + } +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/DistributedIdGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/DistributedIdGenerator.java new file mode 100644 index 00000000..276b55ca --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/DistributedIdGenerator.java @@ -0,0 +1,81 @@ +package io.appform.ranger.discovery.bundle.id.generator; + +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.Id; +import io.appform.ranger.discovery.bundle.id.IdUtils; +import io.appform.ranger.discovery.bundle.id.NonceInfo; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import io.appform.ranger.discovery.bundle.id.nonce.NonceGeneratorType; +import io.appform.ranger.discovery.bundle.id.nonce.PartitionAwareNonceGenerator; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.time.Clock; +import java.util.Optional; +import java.util.function.Function; +import java.util.regex.Pattern; + +@Slf4j +public class DistributedIdGenerator extends IdGeneratorBase { + private static final int MINIMUM_ID_LENGTH = 22; + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyMMddHHmmss"); + private static final Pattern PATTERN = Pattern.compile("(.*)([0-9]{12})([0-9]{3})([0-9]{4})([0-9]{3})"); + + public DistributedIdGenerator(final IdGeneratorConfig idGeneratorConfig, + final Function partitionResolverSupplier, + final NonceGeneratorType nonceGeneratorType, + final MetricRegistry metricRegistry, + final Clock clock, + final IdFormatter idFormatter) { + super(idFormatter, + new PartitionAwareNonceGenerator( + idGeneratorConfig, partitionResolverSupplier, idFormatter, metricRegistry, clock) + ); + } + + public DistributedIdGenerator(final IdGeneratorConfig idGeneratorConfig, + final Function partitionResolverSupplier, + final NonceGeneratorType nonceGeneratorType, + final MetricRegistry metricRegistry, + final Clock clock) { + this(idGeneratorConfig, partitionResolverSupplier, nonceGeneratorType, metricRegistry, clock, IdFormatters.secondPrecision()); + } + + public Id generateForPartition(final String namespace, final int targetPartitionId) { + val idInfo = nonceGenerator.generateForPartition(namespace, targetPartitionId); + return this.getIdFromNonceInfo(idInfo, namespace, idFormatter); + } + + /** + * Generate id by parsing given string + * + * @param idString String idString + * @return ID if it could be generated + */ + public Optional parse(final String idString) { + if (idString == null + || idString.length() < MINIMUM_ID_LENGTH) { + return Optional.empty(); + } + try { + val matcher = PATTERN.matcher(idString); + if (!matcher.find()) { + return Optional.empty(); + } + return Optional.of(Id.builder() + .id(idString) + .node(Integer.parseInt(matcher.group(4))) + .exponent(Integer.parseInt(matcher.group(3))*1000 + Integer.parseInt(matcher.group(5))) + .generatedDate(DATE_TIME_FORMATTER.parseDateTime(matcher.group(2)).toDate()) + .build()); + } catch (Exception e) { + log.warn("Could not parse idString {}", e.getMessage()); + return Optional.empty(); + } + } + +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/IdGeneratorBase.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/IdGeneratorBase.java index a91d91ca..bad6c5ca 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/IdGeneratorBase.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/generator/IdGeneratorBase.java @@ -34,7 +34,7 @@ public class IdGeneratorBase { @Getter - private int nodeId; + private static int NODE_ID; private final List globalConstraints = new ArrayList<>(); private final Map registeredDomains = new ConcurrentHashMap<>(Map.of(Domain.DEFAULT_DOMAIN_NAME, Domain.DEFAULT)); private final FailsafeExecutor retryer; @@ -59,7 +59,7 @@ protected IdGeneratorBase(final IdFormatter idFormatter, public final synchronized void cleanUp() { globalConstraints.clear(); registeredDomains.clear(); - nodeId = 0; + NODE_ID = 0; } public final void registerDomain(final Domain domain) { @@ -83,19 +83,19 @@ public final synchronized void registerDomainSpecificConstraints( .build()); } - public final Id getIdFromIdInfo(final NonceInfo nonceInfo, final String namespace, final IdFormatter idFormatter) { + public final Id getIdFromNonceInfo(final NonceInfo nonceInfo, final String namespace, final IdFormatter idFormatter) { val dateTime = new DateTime(nonceInfo.getTime()); - val id = String.format("%s%s", namespace, idFormatter.format(dateTime, getNodeId(), nonceInfo.getExponent())); + val id = String.format("%s%s", namespace, idFormatter.format(dateTime, getNODE_ID(), nonceInfo.getExponent())); return Id.builder() .id(id) .exponent(nonceInfo.getExponent()) .generatedDate(dateTime.toDate()) - .node(getNodeId()) + .node(getNODE_ID()) .build(); } - public final Id getIdFromIdInfo(final NonceInfo nonceInfo, final String namespace) { - return getIdFromIdInfo(nonceInfo, namespace, idFormatter); + public final Id getIdFromNonceInfo(final NonceInfo nonceInfo, final String namespace) { + return getIdFromNonceInfo(nonceInfo, namespace, idFormatter); } public final IdValidationState validateId(final List inConstraints, final Id id, final boolean skipGlobal) { @@ -136,13 +136,13 @@ public final IdValidationState validateId(final List inC */ public final Id generate(final String namespace) { val idInfo = nonceGenerator.generate(namespace); - return getIdFromIdInfo(idInfo, namespace); + return this.getIdFromNonceInfo(idInfo, namespace, idFormatter); } public final Id generate(final String namespace, final IdFormatter idFormatter) { val idInfo = nonceGenerator.generate(namespace); - return getIdFromIdInfo(idInfo, namespace, idFormatter); + return this.getIdFromNonceInfo(idInfo, namespace, idFormatter); } /** @@ -176,20 +176,21 @@ public final Optional generateWithConstraints(final IdGenerationRequest requ return Optional.ofNullable(retryer.get( () -> { val idInfoOptional = nonceGenerator.generateWithConstraints(idGenerationInput); - val id = getIdFromIdInfo(idInfoOptional, request.getPrefix(), request.getIdFormatter()); + val id = getIdFromNonceInfo(idInfoOptional, request.getPrefix(), request.getIdFormatter()); return new GenerationResult( idInfoOptional, validateId(request.getConstraints(), id, request.isSkipGlobal()), - domain); + domain, + request.getPrefix()); })) .filter(generationResult -> generationResult.getState() == IdValidationState.VALID) - .map(generationResult -> this.getIdFromIdInfo(generationResult.getNonceInfo(), request.getPrefix(), request.getIdFormatter())); + .map(generationResult -> this.getIdFromNonceInfo(generationResult.getNonceInfo(), request.getPrefix(), request.getIdFormatter())); } - public final void setNodeId(int nodeId) { - if (this.nodeId > 0) { + public final void setNODE_ID(int nodeId) { + if (NODE_ID > 0) { throw new RuntimeException("Node ID already set"); } - this.nodeId = nodeId; + NODE_ID = nodeId; } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGenerator.java index 56035f9b..6e96aff1 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGenerator.java @@ -3,15 +3,23 @@ import dev.failsafe.event.ExecutionAttemptedEvent; import io.appform.ranger.discovery.bundle.id.GenerationResult; import io.appform.ranger.discovery.bundle.id.NonceInfo; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; import io.appform.ranger.discovery.bundle.id.request.IdGenerationInput; +import lombok.Getter; import lombok.val; +@Getter public abstract class NonceGenerator { + private IdFormatter idFormatter; protected NonceGenerator() { } + protected NonceGenerator(final IdFormatter idFormatter1) { + this.idFormatter = idFormatter1; + } + public int readRetryCount() { try { val count = Integer.parseInt(System.getenv().getOrDefault("NUM_ID_GENERATION_RETRIES", "512")); @@ -30,12 +38,14 @@ public int readRetryCount() { * Generate id with given namespace * * @param namespace String namespace for ID to be generated - * @return Generated IdInfo + * @return Generated NonceInfo */ public abstract NonceInfo generate(final String namespace); public abstract NonceInfo generateWithConstraints(final IdGenerationInput request); + public abstract NonceInfo generateForPartition(final String namespace, final int targetPartitionId) ; + public abstract void retryEventListener(final ExecutionAttemptedEvent event); } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGeneratorType.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGeneratorType.java new file mode 100644 index 00000000..15402024 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/NonceGeneratorType.java @@ -0,0 +1,6 @@ +package io.appform.ranger.discovery.bundle.id.nonce; + +public enum NonceGeneratorType { + RANDOM, + PARTITION_AWARE +} diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/PartitionAwareNonceGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/PartitionAwareNonceGenerator.java new file mode 100644 index 00000000..e9e74184 --- /dev/null +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/PartitionAwareNonceGenerator.java @@ -0,0 +1,173 @@ +package io.appform.ranger.discovery.bundle.id.nonce; + +import com.codahale.metrics.MetricRegistry; +import dev.failsafe.event.ExecutionAttemptedEvent; +import io.appform.ranger.discovery.bundle.id.GenerationResult; +import io.appform.ranger.discovery.bundle.id.NonceInfo; +import io.appform.ranger.discovery.bundle.id.IdUtils; +import io.appform.ranger.discovery.bundle.id.IdValidationState; +import io.appform.ranger.discovery.bundle.id.PartitionIdTracker; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; +import io.appform.ranger.discovery.bundle.id.config.NamespaceConfig; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatter; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import io.appform.ranger.discovery.bundle.id.request.IdGenerationInput; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + + +@SuppressWarnings("unused") +@Slf4j +@Getter +public class PartitionAwareNonceGenerator extends NonceGenerator { + private final SecureRandom secureRandom = new SecureRandom(Long.toBinaryString(System.currentTimeMillis()).getBytes()); + private final Map idStore = new ConcurrentHashMap<>(); + private final Function partitionResolver; + private final IdGeneratorConfig idGeneratorConfig; + private final MetricRegistry metricRegistry; + private final Clock clock; + + /** idStore Structure + { + namespace: [ + timestamp: { + partitions: [ + { + ids: [], + pointer: int + }, + { + ids: [], + pointer: int + } ... + ], + nextIdCounter: int + } + ] + } + */ + + public PartitionAwareNonceGenerator(final IdGeneratorConfig idGeneratorConfig, + final Function partitionResolverSupplier, + final IdFormatter idFormatter, + final MetricRegistry metricRegistry, + final Clock clock) { + super(idFormatter); + this.idGeneratorConfig = idGeneratorConfig; + this.partitionResolver = partitionResolverSupplier; + this.metricRegistry = metricRegistry; + this.clock = clock; + } + + protected PartitionAwareNonceGenerator(final IdGeneratorConfig idGeneratorConfig, + final Function partitionResolverSupplier, + final MetricRegistry metricRegistry, + final Clock clock) { + this(idGeneratorConfig, partitionResolverSupplier, IdFormatters.secondPrecision(), metricRegistry, clock); + } + + @Override + public NonceInfo generate(final String namespace) { + val targetPartitionId = getTargetPartitionId(); + return generateForPartition(namespace, targetPartitionId); + } + + public NonceInfo generateForPartition(final String namespace, final int targetPartitionId) { + val instant = clock.instant(); + val prefixIdMap = idStore.computeIfAbsent(namespace, k -> getAndInitPartitionIdTrackers(namespace, instant)); + val partitionTracker = getPartitionTracker(prefixIdMap, instant); + val idCounter = generateAndGetId( + partitionTracker, + namespace, + targetPartitionId); + return new NonceInfo(idCounter, partitionTracker.getInstant().getEpochSecond() * 1000L); + } + + @Override + public NonceInfo generateWithConstraints(final IdGenerationInput request) { + val instant = clock.instant(); + val prefixIdMap = idStore.computeIfAbsent(request.getPrefix(), k -> getAndInitPartitionIdTrackers(request.getPrefix(), clock.instant())); + val partitionIdTracker = getPartitionTracker(prefixIdMap, instant); + val targetPartitionId = getTargetPartitionId(); + return generateForPartition(request.getPrefix(), targetPartitionId); + } + + @Override + public void retryEventListener(final ExecutionAttemptedEvent event) { + val result = event.getLastResult(); + if (null != result && !result.getState().equals(IdValidationState.VALID)) { + val instant = clock.instant(); + val prefixIdMap = idStore.computeIfAbsent(result.getNamespace(), k -> getAndInitPartitionIdTrackers(result.getNamespace(), clock.instant())); + val partitionIdTracker = getPartitionTracker(prefixIdMap, instant); + val mappedPartitionId = resolvePartitionId(result.getNamespace(), result.getNonceInfo()); + partitionIdTracker.addId(mappedPartitionId, result.getNonceInfo()); + } + } + + protected int getTargetPartitionId() { + return getSecureRandom().nextInt(idGeneratorConfig.getPartitionCount()); + } + + private Integer generateAndGetId(final PartitionIdTracker partitionIdTracker, + final String namespace, + final int targetPartitionId) { + val idPool = partitionIdTracker.getPartition(targetPartitionId); + Optional idOptional = idPool.getNextId(); + while (idOptional.isEmpty()) { + val nonceInfo = partitionIdTracker.getNonceInfo(); + val mappedPartitionId = resolvePartitionId(namespace, nonceInfo); + partitionIdTracker.addId(mappedPartitionId, nonceInfo); + idOptional = idPool.getNextId(); + } + return idOptional.get(); + } + + private int getIdPoolSize(final String namespace) { + val idPoolSizeOptional = idGeneratorConfig.getNamespaceConfig().stream() + .filter(namespaceConfig -> namespaceConfig.getNamespace().equals(namespace)) + .map(NamespaceConfig::getIdPoolSizePerBucket) + .filter(Objects::nonNull) + .findFirst(); + return idPoolSizeOptional.orElseGet(() -> idGeneratorConfig.getDefaultNamespaceConfig().getIdPoolSizePerPartition()); + } + + private PartitionIdTracker[] getAndInitPartitionIdTrackers(final String namespace, final Instant instant) { + val partitionTrackerList = new PartitionIdTracker[idGeneratorConfig.getDataStorageLimitInSeconds()]; + for (int i = 0; i event) { val result = event.getLastResult(); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java index b7c70371..98020083 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdGeneratorTest.java @@ -181,7 +181,7 @@ void testConstraintFailure() { @Test void testNodeId() { - val generatedId = IdGenerator.generate("TEST123"); + val generatedId = IdGenerator.generate("TEST"); val parsedId = IdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getNode(), nodeId); @@ -225,9 +225,16 @@ void testParseSuccess() { } @Test - void testParseSuccessAfterGeneration() { + void testParseFailAfterGeneration() { val generatedId = IdGenerator.generate("TEST123"); val parsedId = IdGenerator.parse(generatedId.getId()).orElse(null); + Assertions.assertNull(parsedId); + } + + @Test + void testParseSuccessAfterGeneration() { + val generatedId = IdGenerator.generate("TEST"); + val parsedId = IdGenerator.parse(generatedId.getId()).orElse(null); Assertions.assertNotNull(parsedId); Assertions.assertEquals(parsedId.getId(), generatedId.getId()); Assertions.assertEquals(parsedId.getExponent(), generatedId.getExponent()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdParsersTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdParsersTest.java new file mode 100644 index 00000000..0d1d0e06 --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/IdParsersTest.java @@ -0,0 +1,28 @@ +package io.appform.ranger.discovery.bundle.id; + +import io.appform.ranger.discovery.bundle.id.formatter.IdParsers; +import lombok.val; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class IdParsersTest { + + @Test + void testDefaultId() throws ParseException { + val id = "T2407101232336168748798"; + val parsedId = IdParsers.parse(id).orElse(null); + Assertions.assertNotNull(parsedId); + Assertions.assertEquals(id, parsedId.getId()); + Assertions.assertEquals(798, parsedId.getExponent()); + Assertions.assertEquals(8748, parsedId.getNode()); + assertDate("240710123233616", parsedId.getGeneratedDate()); + } + + private void assertDate(final String dateString, final Date date) throws ParseException { + Assertions.assertEquals(new SimpleDateFormat("yyMMddHHmmssSSS").parse(dateString), date); + } +} diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareNonceGeneratorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareNonceGeneratorTest.java new file mode 100644 index 00000000..1be3f073 --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/PartitionAwareNonceGeneratorTest.java @@ -0,0 +1,326 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.discovery.bundle.id.config.DefaultNamespaceConfig; +import io.appform.ranger.discovery.bundle.id.config.IdGeneratorConfig; +import io.appform.ranger.discovery.bundle.id.constraints.IdValidationConstraint; +import io.appform.ranger.discovery.bundle.id.formatter.IdFormatters; +import io.appform.ranger.discovery.bundle.id.nonce.NonceGeneratorType; +import io.appform.ranger.discovery.bundle.id.generator.DistributedIdGenerator; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.math.BigInteger; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +/** + * Test for {@link DistributedIdGenerator} + */ +@Slf4j +class PartitionAwareNonceGeneratorTest { + final int numThreads = 5; + final int iterationCountPerThread = 100000; + final int partitionCount = 1000; + final Function partitionResolverSupplier = (txnId) -> Integer.parseInt(txnId.substring(txnId.length() - 6)) % partitionCount; + private final IdGeneratorConfig idGeneratorConfig = + IdGeneratorConfig.builder() + .partitionCount(partitionCount) + .defaultNamespaceConfig(DefaultNamespaceConfig.builder().idPoolSizePerPartition(100).build()) + .build(); + private DistributedIdGenerator distributedIdGenerator; + private NonceGeneratorType nonceGeneratorType; + private final MetricRegistry metricRegistry = mock(MetricRegistry.class); + + @BeforeEach + void setup() { + nonceGeneratorType = NonceGeneratorType.PARTITION_AWARE; + val meter = mock(Meter.class); + doReturn(meter).when(metricRegistry).meter(anyString()); + doNothing().when(meter).mark(); + distributedIdGenerator = new DistributedIdGenerator( + idGeneratorConfig, partitionResolverSupplier, nonceGeneratorType, metricRegistry, Clock.systemDefaultZone() + ); + distributedIdGenerator.setNODE_ID(9999); + } + + @AfterEach + void cleanup() { + distributedIdGenerator.cleanUp(); + } + + @Test + void testGenerateWithBenchmark() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); + TestUtil.runMTTest( + numThreads, + iterationCountPerThread, + (k) -> { + val id = distributedIdGenerator.generate("P"); + allIdsList.add(id); + }, + true, + this.getClass().getName() + ".testGenerateWithBenchmark"); + Assertions.assertEquals(numThreads * iterationCountPerThread, allIdsList.size()); + checkUniqueIds(allIdsList); + checkDistribution(allIdsList, partitionResolverSupplier, idGeneratorConfig); + } + + @Test + void testGenerateWithConstraints() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); + IdValidationConstraint partitionConstraint = (k) -> k.getExponent() % 4 == 0; + val iterationCount = 50000; + distributedIdGenerator.registerGlobalConstraints(List.of((k) -> k.getExponent() % 4 == 0)); + TestUtil.runMTTest( + numThreads, + iterationCount, + (k) -> { + val id = distributedIdGenerator.generateWithConstraints("P", Domain.DEFAULT_DOMAIN_NAME, false); + id.ifPresent(allIdsList::add); + }, + false, + this.getClass().getName() + ".testGenerateWithConstraints"); + checkUniqueIds(allIdsList); + +// Assert No ID was generated for Invalid partitions + for (val id : allIdsList) { + Assertions.assertTrue(partitionConstraint.isValid(id)); + } + } + + @Test + void testGenerateAccuracy() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); + TestUtil.runMTTest( + numThreads, + iterationCountPerThread, + (k) -> { + val id = distributedIdGenerator.generate("P"); + allIdsList.add(id); + }, + false, + this.getClass().getName() + ".testGenerateAccuracy"); + checkUniqueIds(allIdsList); + checkDistribution(allIdsList, partitionResolverSupplier, idGeneratorConfig); + } + + void checkUniqueIds(List allIdsList) { + val uniqueIdSet = allIdsList.stream() + .map(Id::getId) + .collect(Collectors.toSet()); + Assertions.assertEquals(allIdsList.size(), uniqueIdSet.size()); + } + + protected HashMap getIdCountMap(List allIdsList, Function partitionResolver) { + val idCountMap = new HashMap(); + for (val id : allIdsList) { + val partitionId = partitionResolver.apply(id.getId()); + idCountMap.put(partitionId, idCountMap.getOrDefault(partitionId, 0) + 1); + } + return idCountMap; + } + + protected void checkDistribution(List allIdsList, Function partitionResolver, IdGeneratorConfig config) { + val idCountMap = getIdCountMap(allIdsList, partitionResolver); + val expectedIdCount = (double) allIdsList.size() / config.getPartitionCount(); + for (int partitionId = 0; partitionId < config.getPartitionCount(); partitionId++) { + Assertions.assertTrue(expectedIdCount * 0.8 <= idCountMap.get(partitionId), + String.format("Partition %s generated %s ids, expected was more than: %s", + partitionId, idCountMap.get(partitionId), expectedIdCount * 0.8)); + Assertions.assertTrue(idCountMap.get(partitionId) <= expectedIdCount * 1.2, + String.format("Partition %s generated %s ids, expected was less than: %s", + partitionId, idCountMap.get(partitionId), expectedIdCount * 1.2)); + } + } + + @Test + void testUniqueIds() { + HashSet allIDs = new HashSet<>(); + boolean allIdsUnique = true; + for (int i = 0; i < iterationCountPerThread; i += 1) { + val txnId = distributedIdGenerator.generate("P").getId(); + if (allIDs.contains(txnId)) { + allIdsUnique = false; + break; + } else { + allIDs.add(txnId); + } + } + Assertions.assertTrue(allIdsUnique); + } + + @Test + void testFirstAndLastPartitionInclusion() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); + TestUtil.runMTTest( + numThreads, + iterationCountPerThread, + (k) -> { + val id = distributedIdGenerator.generate("P"); + allIdsList.add(id); + }, + false, + this.getClass().getName() + ".testGenerateAccuracy"); + + val idCountMap = getIdCountMap(allIdsList, partitionResolverSupplier); + Assertions.assertTrue(idCountMap.get(0) > 0); + Assertions.assertTrue(idCountMap.get(partitionCount - 1) > 0); + } + + @Test + void testDataReset() throws IOException { + val allIdsList = Collections.synchronizedList(new ArrayList()); + val clock = mock(Clock.class); + doReturn(Instant.parse("2000-01-01T00:00:00Z")).when(clock).instant(); + val distributedIdGenerator = new DistributedIdGenerator(idGeneratorConfig, partitionResolverSupplier, nonceGeneratorType, metricRegistry, clock); + TestUtil.runMTTest( + numThreads, + iterationCountPerThread, + (k) -> { + val id = distributedIdGenerator.generate("P"); + allIdsList.add(id); + }, + false, null); + doReturn(Instant.parse("2000-01-01T00:01:00Z")).when(clock).instant(); + TestUtil.runMTTest( + numThreads, + iterationCountPerThread, + (k) -> { + val id = distributedIdGenerator.generate("P"); + allIdsList.add(id); + }, + false, null); + Assertions.assertEquals(2 * numThreads * iterationCountPerThread, allIdsList.size()); + checkUniqueIds(allIdsList); + } + + @Test + void testComputation() throws IOException { + val partitionResolverCount = new AtomicInteger(0); + Function partitionResolverSupplierWithCount = (t) -> { + partitionResolverCount.incrementAndGet(); + return partitionResolverSupplier.apply(t); + }; + val localdistributedIdGenerator = new DistributedIdGenerator(idGeneratorConfig, partitionResolverSupplierWithCount, nonceGeneratorType, metricRegistry, Clock.systemDefaultZone()); + val allIdsList = Collections.synchronizedList(new ArrayList()); + TestUtil.runMTTest( + numThreads, + iterationCountPerThread, + (k) -> { + val id = localdistributedIdGenerator.generate("P"); + allIdsList.add(id); + }, + false, + null); + val expectedIdCount = numThreads * iterationCountPerThread; + Assertions.assertEquals(expectedIdCount, allIdsList.size()); + checkUniqueIds(allIdsList); + log.debug("partitionResolverSupplier was called {} times - expected count was: {}", partitionResolverCount.get(), expectedIdCount); + } + + @Test + void testGenerateOriginal() { + distributedIdGenerator = new DistributedIdGenerator( + idGeneratorConfig, partitionResolverSupplier, nonceGeneratorType, metricRegistry, Clock.systemDefaultZone() + ); + val idOptional = distributedIdGenerator.generate("TEST"); + String id = idOptional.getId(); + Assertions.assertEquals(26, id.length()); + } + + @Test + void testGenerateBase36() { + val distributedIdGeneratorLocal = new DistributedIdGenerator( + idGeneratorConfig, + (txnId) -> new BigInteger(txnId.substring(txnId.length() - 6), 36).abs().intValue() % partitionCount, + nonceGeneratorType, + metricRegistry, Clock.systemDefaultZone(), IdFormatters.base36() + ); + String id = distributedIdGeneratorLocal.generate("TEST").getId(); + Assertions.assertEquals(18, id.length()); + } + + @Test + void testConstraintFailure() { + val domainName = "ALL_INVALID"; + distributedIdGenerator.registerDomain( + Domain.builder() + .domain(domainName) + .constraints(List.of(id -> false)) + .build() + ); + Assertions.assertFalse(distributedIdGenerator.generateWithConstraints( + "TST", + domainName, + true).isPresent()); + } + + @Test + void testParseFailure() { + //Null or Empty String + Assertions.assertFalse(distributedIdGenerator.parse(null).isPresent()); + Assertions.assertFalse(distributedIdGenerator.parse("").isPresent()); + + //Invalid length + Assertions.assertFalse(distributedIdGenerator.parse("TEST").isPresent()); + + //Invalid chars + Assertions.assertFalse(distributedIdGenerator.parse("XCL983dfb1ee0a847cd9e7321fcabc2f223").isPresent()); + Assertions.assertFalse(distributedIdGenerator.parse("XCL98-3df-b1e:e0a847cd9e7321fcabc2f223").isPresent()); + + //Invalid month + Assertions.assertFalse(distributedIdGenerator.parse("ABC2032250959030643972247").isPresent()); + //Invalid date + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011450959030643972247").isPresent()); + //Invalid hour + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011259659030643972247").isPresent()); + //Invalid minute + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011250972030643972247").isPresent()); + //Invalid sec + Assertions.assertFalse(distributedIdGenerator.parse("ABC2011250959720643972247").isPresent()); + } + + @Test + void testParseSuccess() { + val idString = "ABC2011250959031643972247"; + val id = distributedIdGenerator.parse(idString).orElse(null); + Assertions.assertNotNull(id); + Assertions.assertEquals(idString, id.getId()); + Assertions.assertEquals(164247, id.getExponent()); + Assertions.assertEquals(3972, id.getNode()); + Assertions.assertEquals(TestUtil.generateDate(2020, 11, 25, 9, 59, 3, 0, ZoneId.systemDefault()), + id.getGeneratedDate()); + } + + @Test + void testParseSuccessAfterGeneration() { + val generatedId = distributedIdGenerator.generate("TEST123"); + val parsedId = distributedIdGenerator.parse(generatedId.getId()).orElse(null); + Assertions.assertNotNull(parsedId); + Assertions.assertEquals(parsedId.getId(), generatedId.getId()); + Assertions.assertEquals(parsedId.getExponent(), generatedId.getExponent()); + Assertions.assertEquals(parsedId.getNode(), generatedId.getNode()); + } + +} \ No newline at end of file diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java new file mode 100644 index 00000000..64491036 --- /dev/null +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/TestUtil.java @@ -0,0 +1,84 @@ +package io.appform.ranger.discovery.bundle.id; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.logback.shaded.guava.base.Stopwatch; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@Slf4j +@UtilityClass +public class TestUtil { + private static final String OUTPUT_PATH = "perf/results/%s.json"; + + public double runMTTest(int numThreads, int iterationCount, final Consumer supplier, final boolean save_output, final String outputFileName) throws IOException { + final ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + final List> futures = IntStream.range(0, numThreads) + .mapToObj(i -> executorService.submit(() -> { + final Stopwatch stopwatch = Stopwatch.createStarted(); + IntStream.range(0, iterationCount).forEach(supplier::accept); + return stopwatch.elapsed(TimeUnit.MILLISECONDS); + })) + .collect(Collectors.toList()); + final long total = futures.stream() + .mapToLong(f -> { + try { + return f.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return 0; + } catch (ExecutionException e) { + throw new IllegalStateException(e); + } + }) + .sum(); + if (save_output){ + writeToFile(numThreads, iterationCount, total, outputFileName); + } + log.info("Finished Execution for {} iterations in avg time: {}", iterationCount, ((double) total) / numThreads); + return total; + } + + public void writeToFile(int numThreads, int iterationCount, long totalMillis, String outputFileName) throws IOException { + val mapper = new ObjectMapper(); + val outputFilePath = Paths.get(String.format(OUTPUT_PATH, outputFileName)); + val outputNode = mapper.createObjectNode(); + outputNode.put("name", outputFileName); + outputNode.put("iterations", iterationCount); + outputNode.put("threads", numThreads); + outputNode.put("totalMillis", totalMillis); + outputNode.put("avgTime", ((double) totalMillis) / numThreads); + Files.write(outputFilePath, mapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(outputNode)); + } + + public Date generateDate(int year, int month, int day, int hour, int min, int sec, int ms, ZoneId zoneId) { + return Date.from( + Instant.from( + ZonedDateTime.of( + LocalDateTime.of( + year, month, day, hour, min, sec, Math.multiplyExact(ms, 1000000) + ), + zoneId + ) + ) + ); + } +}