Skip to content

Commit 0f930c9

Browse files
mneethirajmneethiraj
committed
RANGER-4100: Efficient computation of the smallest set of evaluators returned by search of multiple Trie trees - #2
(cherry picked from commit 93e888f)
1 parent 63ba221 commit 0f930c9

File tree

3 files changed

+52
-111
lines changed

3 files changed

+52
-111
lines changed

‎agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java‎

Lines changed: 47 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator.RangerPolicyResourceEvaluator;
4343
import org.apache.ranger.plugin.store.AbstractServiceStore;
4444
import org.apache.ranger.plugin.util.RangerPerfTracer;
45+
import org.apache.ranger.plugin.util.RangerResourceEvaluatorsRetriever;
4546
import org.apache.ranger.plugin.util.ServiceDefUtil;
4647
import org.apache.ranger.plugin.util.ServicePolicies;
4748
import org.slf4j.Logger;
@@ -72,30 +73,30 @@ enum AuditModeEnum {
7273
AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
7374
}
7475

75-
private final String serviceName;
76-
private final String zoneName;
77-
private final String appId;
78-
private final RangerPolicyEngineOptions options;
79-
private final RangerPluginContext pluginContext;
80-
private final RangerServiceDef serviceDef;
81-
private /*final*/ List<RangerPolicy> policies;
82-
private final long policyVersion;
83-
private /*final*/ List<RangerContextEnricher> contextEnrichers;
84-
private final AuditModeEnum auditModeEnum;
85-
private final Map<String, AuditInfo> accessAuditCache;
86-
private final String componentServiceName;
87-
private final RangerServiceDef componentServiceDef;
88-
private final Map<String, RangerResourceTrie> policyResourceTrie;
89-
private final Map<String, RangerResourceTrie> dataMaskResourceTrie;
90-
private final Map<String, RangerResourceTrie> rowFilterResourceTrie;
91-
private final Map<String, RangerResourceTrie> auditFilterResourceTrie;
92-
private List<RangerPolicyEvaluator> policyEvaluators;
93-
private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
94-
private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
95-
private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
96-
private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
97-
private boolean isContextEnrichersShared = false;
98-
private boolean isPreCleaned = false;
76+
private final String serviceName;
77+
private final String zoneName;
78+
private final String appId;
79+
private final RangerPolicyEngineOptions options;
80+
private final RangerPluginContext pluginContext;
81+
private final RangerServiceDef serviceDef;
82+
private /*final*/ List<RangerPolicy> policies;
83+
private final long policyVersion;
84+
private /*final*/ List<RangerContextEnricher> contextEnrichers;
85+
private final AuditModeEnum auditModeEnum;
86+
private final Map<String, AuditInfo> accessAuditCache;
87+
private final String componentServiceName;
88+
private final RangerServiceDef componentServiceDef;
89+
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> policyResourceTrie;
90+
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> dataMaskResourceTrie;
91+
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> rowFilterResourceTrie;
92+
private final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> auditFilterResourceTrie;
93+
private List<RangerPolicyEvaluator> policyEvaluators;
94+
private List<RangerPolicyEvaluator> dataMaskPolicyEvaluators;
95+
private List<RangerPolicyEvaluator> rowFilterPolicyEvaluators;
96+
private final List<RangerPolicyEvaluator> auditPolicyEvaluators;
97+
private Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
98+
private boolean isContextEnrichersShared = false;
99+
private boolean isPreCleaned = false;
99100

100101
RangerPolicyRepository(final RangerPolicyRepository other, final List<RangerPolicyDelta> deltas, long policyVersion) {
101102
this.serviceName = other.serviceName;
@@ -117,8 +118,8 @@ enum AuditModeEnum {
117118
if (other.policyResourceTrie != null) {
118119
this.policyResourceTrie = new HashMap<>();
119120

120-
for (Map.Entry<String, RangerResourceTrie> entry : other.policyResourceTrie.entrySet()) {
121-
policyResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
121+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.policyResourceTrie.entrySet()) {
122+
policyResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
122123
}
123124
} else {
124125
this.policyResourceTrie = null;
@@ -127,8 +128,8 @@ enum AuditModeEnum {
127128
if (other.dataMaskResourceTrie != null) {
128129
this.dataMaskResourceTrie = new HashMap<>();
129130

130-
for (Map.Entry<String, RangerResourceTrie> entry : other.dataMaskResourceTrie.entrySet()) {
131-
dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
131+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.dataMaskResourceTrie.entrySet()) {
132+
dataMaskResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
132133
}
133134
} else {
134135
this.dataMaskResourceTrie = null;
@@ -137,8 +138,8 @@ enum AuditModeEnum {
137138
if (other.rowFilterResourceTrie != null) {
138139
this.rowFilterResourceTrie = new HashMap<>();
139140

140-
for (Map.Entry<String, RangerResourceTrie> entry : other.rowFilterResourceTrie.entrySet()) {
141-
rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
141+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.rowFilterResourceTrie.entrySet()) {
142+
rowFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
142143
}
143144
} else {
144145
this.rowFilterResourceTrie = null;
@@ -147,8 +148,8 @@ enum AuditModeEnum {
147148
if (other.auditFilterResourceTrie != null) {
148149
this.auditFilterResourceTrie = new HashMap<>();
149150

150-
for (Map.Entry<String, RangerResourceTrie> entry : other.auditFilterResourceTrie.entrySet()) {
151-
auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie(entry.getValue()));
151+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : other.auditFilterResourceTrie.entrySet()) {
152+
auditFilterResourceTrie.put(entry.getKey(), new RangerResourceTrie<>(entry.getValue()));
152153
}
153154
} else {
154155
this.auditFilterResourceTrie = null;
@@ -157,7 +158,7 @@ enum AuditModeEnum {
157158
if (other.accessAuditCache != null) {
158159
int auditResultCacheSize = other.accessAuditCache.size();
159160

160-
this.accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
161+
this.accessAuditCache = Collections.synchronizedMap(new CacheMap<>(auditResultCacheSize));
161162
} else {
162163
this.accessAuditCache = null;
163164
}
@@ -217,7 +218,7 @@ public RangerPolicyRepository(ServicePolicies servicePolicies, RangerPluginConte
217218
final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024;
218219

219220
int auditResultCacheSize = pluginContext.getConfig().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE);
220-
accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
221+
accessAuditCache = Collections.synchronizedMap(new CacheMap<>(auditResultCacheSize));
221222
} else {
222223
accessAuditCache = null;
223224
}
@@ -587,7 +588,7 @@ List<PolicyEvaluatorForTag> getLikelyMatchPolicyEvaluators(RangerAccessRequest r
587588

588589
if (CollectionUtils.isNotEmpty(tags) && getServiceDef() != null) {
589590

590-
ret = new ArrayList<PolicyEvaluatorForTag>();
591+
ret = new ArrayList<>();
591592

592593
for (RangerTagForEval tag : tags) {
593594
if (tag.isApplicable(accessTime)) {
@@ -691,7 +692,7 @@ List<RangerPolicyEvaluator> getLikelyMatchAuditPolicyEvaluators(RangerAccessRequ
691692
return auditFilterResourceTrie == null || StringUtils.isEmpty(resourceStr) ? getAuditPolicyEvaluators() : getLikelyMatchPolicyEvaluators(auditFilterResourceTrie, request);
692693
}
693694

694-
private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie> resourceTrie, RangerAccessRequest request) {
695+
private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> resourceTrie, RangerAccessRequest request) {
695696
List<RangerPolicyEvaluator> ret = Collections.EMPTY_LIST;
696697

697698
RangerAccessResource resource = request.getResource();
@@ -702,67 +703,7 @@ private List<RangerPolicyEvaluator> getLikelyMatchPolicyEvaluators(Map<String, R
702703
perf = RangerPerfTracer.getPerfTracer(PERF_TRIE_OP_LOG, "RangerPolicyRepository.getLikelyMatchEvaluators(resource=" + resource.getAsString() + ")");
703704
}
704705

705-
List<String> resourceKeys = resource == null ? null : options.getServiceDefHelper().getOrderedResourceNames(resource.getKeys());
706-
Set<RangerPolicyResourceEvaluator> smallestList = null;
707-
708-
if (CollectionUtils.isNotEmpty(resourceKeys)) {
709-
710-
for (String resourceName : resourceKeys) {
711-
RangerResourceTrie<RangerPolicyResourceEvaluator> trie = resourceTrie.get(resourceName);
712-
713-
if (trie == null) { // if no trie exists for this resource level, ignore and continue to next level
714-
continue;
715-
}
716-
717-
Set<RangerPolicyResourceEvaluator> serviceResourceMatchersForResource = trie.getEvaluatorsForResource(resource.getValue(resourceName), request.getResourceMatchingScope());
718-
Set<RangerPolicyResourceEvaluator> inheritedResourceMatchers = trie.getInheritedEvaluators();
719-
720-
if (smallestList != null) {
721-
if (CollectionUtils.isEmpty(inheritedResourceMatchers) && CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
722-
smallestList = null;
723-
} else if (CollectionUtils.isEmpty(inheritedResourceMatchers)) {
724-
smallestList.retainAll(serviceResourceMatchersForResource);
725-
} else if (CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
726-
smallestList.retainAll(inheritedResourceMatchers);
727-
} else {
728-
Set<RangerPolicyResourceEvaluator> smaller, bigger;
729-
if (serviceResourceMatchersForResource.size() < inheritedResourceMatchers.size()) {
730-
smaller = serviceResourceMatchersForResource;
731-
bigger = inheritedResourceMatchers;
732-
} else {
733-
smaller = inheritedResourceMatchers;
734-
bigger = serviceResourceMatchersForResource;
735-
}
736-
Set<RangerPolicyResourceEvaluator> tmp = new HashSet<>();
737-
if (smallestList.size() < smaller.size()) {
738-
smallestList.stream().filter(smaller::contains).forEach(tmp::add);
739-
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
740-
} else {
741-
smaller.stream().filter(smallestList::contains).forEach(tmp::add);
742-
if (smallestList.size() < bigger.size()) {
743-
smallestList.stream().filter(bigger::contains).forEach(tmp::add);
744-
} else {
745-
bigger.stream().filter(smallestList::contains).forEach(tmp::add);
746-
}
747-
}
748-
smallestList = tmp;
749-
}
750-
} else {
751-
if (CollectionUtils.isEmpty(inheritedResourceMatchers) || CollectionUtils.isEmpty(serviceResourceMatchersForResource)) {
752-
Set<RangerPolicyResourceEvaluator> tmp = CollectionUtils.isEmpty(inheritedResourceMatchers) ? serviceResourceMatchersForResource : inheritedResourceMatchers;
753-
smallestList = resourceKeys.size() == 1 || CollectionUtils.isEmpty(tmp) ? tmp : new HashSet<>(tmp);
754-
} else {
755-
smallestList = new HashSet<>(serviceResourceMatchersForResource);
756-
smallestList.addAll(inheritedResourceMatchers);
757-
}
758-
}
759-
760-
if (CollectionUtils.isEmpty(smallestList)) {// no tags for this resource, bail out
761-
smallestList = null;
762-
break;
763-
}
764-
}
765-
}
706+
Collection<RangerPolicyResourceEvaluator> smallestList = RangerResourceEvaluatorsRetriever.getEvaluators(resourceTrie, resource.getAsMap(), request.getResourceMatchingScope());
766707

767708
if (smallestList != null) {
768709
if (smallestList.size() == 0) {
@@ -1224,8 +1165,8 @@ private List<RangerPolicyEvaluator> getReorderedPolicyEvaluators(List<RangerPoli
12241165
return ret;
12251166
}
12261167

1227-
private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
1228-
final Map<String, RangerResourceTrie> ret;
1168+
private Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> createResourceTrieMap(List<? extends RangerPolicyEvaluator> evaluators, boolean optimizeTrieForRetrieval, boolean optimizeTrieForSpace) {
1169+
final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> ret;
12291170

12301171
if (serviceDef != null && CollectionUtils.isNotEmpty(serviceDef.getResources())) {
12311172
ret = new HashMap<>();
@@ -1240,7 +1181,7 @@ private Map<String, RangerResourceTrie> createResourceTrieMap(List<? extends Ran
12401181
return ret;
12411182
}
12421183

1243-
private void updateTrie(Map<String, RangerResourceTrie> trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
1184+
private void updateTrie(Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap, Integer policyDeltaType, RangerPolicyEvaluator oldEvaluator, RangerPolicyEvaluator newEvaluator) {
12441185
if (LOG.isDebugEnabled()) {
12451186
LOG.debug("==> RangerPolicyRepository.updateTrie(policyDeltaType=" + policyDeltaType + "): ");
12461187
}
@@ -1436,7 +1377,7 @@ private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final Ranger
14361377
break;
14371378
}
14381379

1439-
Map<String, RangerResourceTrie> trieMap = getTrie(policyType);
1380+
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trieMap = getTrie(policyType);
14401381

14411382
if (trieMap != null) {
14421383
updateTrie(trieMap, changeType, currentEvaluator, newEvaluator);
@@ -1457,8 +1398,8 @@ private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final Ranger
14571398
return ret;
14581399
}
14591400

1460-
Map<String, RangerResourceTrie> getTrie(final int policyType) {
1461-
final Map<String, RangerResourceTrie> ret;
1401+
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> getTrie(final int policyType) {
1402+
final Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> ret;
14621403
switch (policyType) {
14631404
case RangerPolicy.POLICY_TYPE_ACCESS:
14641405
ret = policyResourceTrie;
@@ -1583,18 +1524,18 @@ private void updateResourceTrie(List<RangerPolicyDelta> deltas) {
15831524

15841525
for (int policyType = 0; policyType < flags.length; policyType++) {
15851526
if (flags[policyType]) {
1586-
Map<String, RangerResourceTrie> trie = getTrie(policyType);
1527+
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> trie = getTrie(policyType);
15871528

15881529
if (trie != null) {
1589-
for (Map.Entry<String, RangerResourceTrie> entry : trie.entrySet()) {
1530+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : trie.entrySet()) {
15901531
entry.getValue().wrapUpUpdate();
15911532
}
15921533
}
15931534
}
15941535
}
15951536

15961537
if (auditFilterResourceTrie != null) {
1597-
for (Map.Entry<String, RangerResourceTrie> entry : auditFilterResourceTrie.entrySet()) {
1538+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : auditFilterResourceTrie.entrySet()) {
15981539
entry.getValue().wrapUpUpdate();
15991540
}
16001541
}

‎agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerResourceTrie.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boo
105105
this(resourceDef, evaluators, isOptimizedForRetrieval, false, pluginContext);
106106
}
107107

108-
public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
108+
public <T extends RangerResourceEvaluator, E> RangerResourceTrie(RangerResourceDef resourceDef, List<E> evaluators, boolean isOptimizedForRetrieval, boolean isOptimizedForSpace, RangerPluginContext pluginContext) {
109109
if(LOG.isDebugEnabled()) {
110110
LOG.debug("==> RangerResourceTrie(" + resourceDef.getName() + ", evaluatorCount=" + evaluators.size() + ", isOptimizedForRetrieval=" + isOptimizedForRetrieval + ", isOptimizedForSpace=" + isOptimizedForSpace + ")");
111111
}
@@ -154,7 +154,7 @@ public RangerResourceTrie(RangerResourceDef resourceDef, List<T> evaluators, boo
154154
this.isOptimizedForRetrieval = !isOptimizedForSpace && isOptimizedForRetrieval; // isOptimizedForSpace takes precedence
155155
this.separatorChar = ServiceDefUtil.getCharOption(matcherOptions, OPTION_PATH_SEPARATOR, DEFAULT_PATH_SEPARATOR_CHAR);
156156

157-
TrieNode<T> tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);
157+
final TrieNode tmpRoot = buildTrie(resourceDef, evaluators, builderThreadCount);
158158

159159
if (builderThreadCount > 1 && tmpRoot == null) { // if multi-threaded trie-creation failed, build using a single thread
160160
this.root = buildTrie(resourceDef, evaluators, 1);

‎agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -995,13 +995,13 @@ public static boolean compare(RangerPolicyRepository me, RangerPolicyRepository
995995
public static boolean compareTrie(final int policyType, RangerPolicyRepository me, RangerPolicyRepository other) {
996996
boolean ret;
997997

998-
Map<String, RangerResourceTrie> myTrie = me.getTrie(policyType);
999-
Map<String, RangerResourceTrie> otherTrie = other.getTrie(policyType);
998+
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> myTrie = me.getTrie(policyType);
999+
Map<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> otherTrie = other.getTrie(policyType);
10001000

10011001
ret = myTrie.size() == otherTrie.size();
10021002

10031003
if (ret) {
1004-
for (Map.Entry<String, RangerResourceTrie> entry : myTrie.entrySet()) {
1004+
for (Map.Entry<String, RangerResourceTrie<RangerPolicyResourceEvaluator>> entry : myTrie.entrySet()) {
10051005
RangerResourceTrie myResourceTrie = entry.getValue();
10061006
RangerResourceTrie otherResourceTrie = otherTrie.get(entry.getKey());
10071007

0 commit comments

Comments
 (0)