Skip to content

Commit b1920de

Browse files
mneethirajmneethiraj
committed
RANGER-3955: eliminate duplicate strings in RangerUserStore, ServiceTags, ServicePolicies
1 parent e35fe23 commit b1920de

File tree

18 files changed

+896
-19
lines changed

18 files changed

+896
-19
lines changed

‎agents-common/src/main/java/org/apache/ranger/authorization/utils/StringUtil.java‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@
3232
import java.util.Set;
3333
import java.util.TimeZone;
3434

35+
import org.apache.commons.collections.CollectionUtils;
36+
import org.apache.commons.collections.MapUtils;
3537
import org.apache.commons.lang.StringUtils;
38+
import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
3639

3740
public class StringUtil {
3841

@@ -338,4 +341,174 @@ public static List<String> getURLs(String configURLs) {
338341
}
339342
return configuredURLs;
340343
}
344+
345+
public static Map<String, Map<String, String>> dedupStringsMapOfMap(Map<String, Map<String, String>> value, Map<String, String> strTbl) {
346+
final Map<String, Map<String, String>> ret;
347+
348+
if (MapUtils.isNotEmpty(value)) {
349+
ret = new HashMap<>(value.size());
350+
351+
for (Map.Entry<String, Map<String, String>> entry : value.entrySet()) {
352+
ret.put(dedupString(entry.getKey(), strTbl), dedupStringsMap(entry.getValue(), strTbl));
353+
}
354+
} else {
355+
ret = value;
356+
}
357+
358+
return ret;
359+
}
360+
361+
public static Map<String, Set<String>> dedupStringsMapOfSet(Map<String, Set<String>> value, Map<String, String> strTbl) {
362+
final Map<String, Set<String>> ret;
363+
364+
if (MapUtils.isNotEmpty(value)) {
365+
ret = new HashMap<>(value.size());
366+
367+
for (Map.Entry<String, Set<String>> entry : value.entrySet()) {
368+
ret.put(dedupString(entry.getKey(), strTbl), dedupStringsSet(entry.getValue(), strTbl));
369+
}
370+
} else {
371+
ret = value;
372+
}
373+
374+
return ret;
375+
}
376+
377+
public static Map<String, List<String>> dedupStringsMapOfList(Map<String, List<String>> value, Map<String, String> strTbl) {
378+
final Map<String, List<String>> ret;
379+
380+
if (MapUtils.isNotEmpty(value)) {
381+
ret = new HashMap<>(value.size());
382+
383+
for (Map.Entry<String, List<String>> entry : value.entrySet()) {
384+
ret.put(dedupString(entry.getKey(), strTbl), dedupStringsList(entry.getValue(), strTbl));
385+
}
386+
} else {
387+
ret = value;
388+
}
389+
390+
return ret;
391+
}
392+
393+
public static HashMap<String, List<String>> dedupStringsHashMapOfList(HashMap<String, List<String>> value, Map<String, String> strTbl) {
394+
final HashMap<String, List<String>> ret;
395+
396+
if (MapUtils.isNotEmpty(value)) {
397+
ret = new HashMap<>(value.size());
398+
399+
for (Map.Entry<String, List<String>> entry : value.entrySet()) {
400+
ret.put(dedupString(entry.getKey(), strTbl), dedupStringsList(entry.getValue(), strTbl));
401+
}
402+
} else {
403+
ret = value;
404+
}
405+
406+
return ret;
407+
}
408+
409+
public static Map<String, Object> dedupStringsMapOfObject(Map<String, Object> value, Map<String, String> strTbl) {
410+
final Map<String, Object> ret;
411+
412+
if (MapUtils.isNotEmpty(value)) {
413+
ret = new HashMap<>(value.size());
414+
415+
for (Map.Entry<String, Object> entry : value.entrySet()) {
416+
ret.put(dedupString(entry.getKey(), strTbl), entry.getValue());
417+
}
418+
} else {
419+
ret = value;
420+
}
421+
422+
return ret;
423+
}
424+
425+
public static Map<String, RangerPolicyResource> dedupStringsMapOfPolicyResource(Map<String, RangerPolicyResource> value, Map<String, String> strTbl) {
426+
final Map<String, RangerPolicyResource> ret;
427+
428+
if (MapUtils.isNotEmpty(value)) {
429+
ret = new HashMap<>(value.size());
430+
431+
for (Map.Entry<String, RangerPolicyResource> entry : value.entrySet()) {
432+
RangerPolicyResource resource = entry.getValue();
433+
434+
resource.dedupStrings(strTbl);
435+
436+
ret.put(dedupString(entry.getKey(), strTbl), resource);
437+
}
438+
} else {
439+
ret = value;
440+
}
441+
442+
return ret;
443+
}
444+
445+
public static Map<String, String> dedupStringsMap(Map<String, String> value, Map<String, String> strTbl) {
446+
final Map<String, String> ret;
447+
448+
if (MapUtils.isNotEmpty(value)) {
449+
ret = new HashMap<>(value.size());
450+
451+
for (Map.Entry<String, String> entry : value.entrySet()) {
452+
ret.put(dedupString(entry.getKey(), strTbl), dedupString(entry.getValue(), strTbl));
453+
}
454+
} else {
455+
ret = value;
456+
}
457+
458+
return ret;
459+
}
460+
461+
public static Set<String> dedupStringsSet(Set<String> value, Map<String, String> strTbl) {
462+
final Set<String> ret;
463+
464+
if (CollectionUtils.isNotEmpty(value)) {
465+
ret = new HashSet<>(value.size());
466+
467+
for (String val : value) {
468+
ret.add(dedupString(val, strTbl));
469+
}
470+
} else {
471+
ret = value;
472+
}
473+
474+
return ret;
475+
}
476+
477+
public static List<String> dedupStringsList(List<String> value, Map<String, String> strTbl) {
478+
final List<String> ret;
479+
480+
if (CollectionUtils.isNotEmpty(value)) {
481+
ret = new ArrayList<>(value.size());
482+
483+
for (String val : value) {
484+
ret.add(dedupString(val, strTbl));
485+
}
486+
} else {
487+
ret = value;
488+
}
489+
490+
return ret;
491+
}
492+
493+
public static Collection<String> dedupStringsCollection(Collection<String> value, Map<String, String> strTbl) {
494+
final Collection<String> ret;
495+
496+
if (CollectionUtils.isNotEmpty(value)) {
497+
ret = value instanceof Set ? new HashSet<>(value.size()) : new ArrayList<>(value.size());
498+
499+
for (String val : value) {
500+
ret.add(dedupString(val, strTbl));
501+
}
502+
} else {
503+
ret = value;
504+
}
505+
506+
return ret;
507+
}
508+
509+
public static String dedupString(String str, Map<String, String> strTbl) {
510+
String ret = str != null ? strTbl.putIfAbsent(str, str) : null;
511+
512+
return ret == null ? str : ret;
513+
}
341514
}

‎agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher {
8585
private boolean disableTrieLookupPrefilter;
8686
private EnrichedServiceTags enrichedServiceTags;
8787
private boolean disableCacheIfServiceNotFound = true;
88+
private boolean dedupStrings = true;
8889

8990
private final BlockingQueue<DownloadTrigger> tagDownloadQueue = new LinkedBlockingQueue<>();
9091
private Timer tagDownloadTimer;
@@ -100,13 +101,13 @@ public void init() {
100101

101102
super.init();
102103

104+
String propertyPrefix = "ranger.plugin." + serviceDef.getName();
103105
String tagRetrieverClassName = getOption(TAG_RETRIEVER_CLASSNAME_OPTION);
106+
long pollingIntervalMs = getLongOption(TAG_REFRESHER_POLLINGINTERVAL_OPTION, 60 * 1000);
104107

105-
long pollingIntervalMs = getLongOption(TAG_REFRESHER_POLLINGINTERVAL_OPTION, 60 * 1000);
106-
108+
dedupStrings = getBooleanConfig(propertyPrefix + ".dedup.strings", true);
107109
disableTrieLookupPrefilter = getBooleanOption(TAG_DISABLE_TRIE_PREFILTER_OPTION, false);
108-
109-
serviceDefHelper = new RangerServiceDefHelper(serviceDef, false);
110+
serviceDefHelper = new RangerServiceDefHelper(serviceDef, false);
110111

111112
if (StringUtils.isNotBlank(tagRetrieverClassName)) {
112113

@@ -127,7 +128,6 @@ public void init() {
127128
}
128129

129130
if (tagRetriever != null) {
130-
String propertyPrefix = "ranger.plugin." + serviceDef.getName();
131131
disableCacheIfServiceNotFound = getBooleanConfig(propertyPrefix + ".disable.cache.if.servicenotfound", true);
132132
String cacheDir = getConfig(propertyPrefix + ".policy.cache.dir", null);
133133
String cacheFilename = String.format("%s_%s_tag.json", appId, serviceName);
@@ -303,6 +303,10 @@ protected void setServiceTags(final ServiceTags serviceTags, final boolean rebui
303303
LOG.info("ServiceTags is null for service " + serviceName);
304304
enrichedServiceTags = null;
305305
} else {
306+
if (dedupStrings) {
307+
serviceTags.dedupStrings();
308+
}
309+
306310
if (!serviceTags.getIsDelta()) {
307311
processServiceTags(serviceTags);
308312
} else {

‎agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerUserStoreEnricher.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public class RangerUserStoreEnricher extends RangerAbstractContextEnricher {
4545
private RangerUserStoreRetriever userStoreRetriever;
4646
private RangerUserStore rangerUserStore;
4747
private boolean disableCacheIfServiceNotFound = true;
48+
private boolean dedupStrings = true;
4849
private final BlockingQueue<DownloadTrigger> userStoreDownloadQueue = new LinkedBlockingQueue<>();
4950
private Timer userStoreDownloadTimer;
5051

@@ -57,8 +58,9 @@ public void init() {
5758
super.init();
5859

5960
String userStoreRetrieverClassName = getOption(USERSTORE_RETRIEVER_CLASSNAME_OPTION);
61+
long pollingIntervalMs = getLongOption(USERSTORE_REFRESHER_POLLINGINTERVAL_OPTION, 3600 * 1000);
6062

61-
long pollingIntervalMs = getLongOption(USERSTORE_REFRESHER_POLLINGINTERVAL_OPTION, 3600 * 1000);
63+
dedupStrings = getBooleanConfig("ranger.plugin." + serviceDef.getName() + ".dedup.strings", true);
6264

6365
if (StringUtils.isNotBlank(userStoreRetrieverClassName)) {
6466

@@ -191,7 +193,12 @@ public void setRangerUserStore(final RangerUserStore rangerUserStore) {
191193
perf = RangerPerfTracer.getPerfTracer(PERF_SET_USERSTORE_LOG, "RangerUserStoreEnricher.setRangerUserStore(newUserStoreVersion=" + rangerUserStore.getUserStoreVersion() + ")");
192194
}
193195

196+
if (dedupStrings) {
197+
rangerUserStore.dedupStrings();
198+
}
199+
194200
this.rangerUserStore = rangerUserStore;
201+
195202
RangerPerfTracer.logAlways(perf);
196203
}
197204

‎agents-common/src/main/java/org/apache/ranger/plugin/model/RangerBaseModelObject.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
package org.apache.ranger.plugin.model;
2121

2222
import java.util.Date;
23+
import java.util.Map;
2324

2425
import javax.xml.bind.annotation.XmlAccessType;
2526
import javax.xml.bind.annotation.XmlAccessorType;
2627
import javax.xml.bind.annotation.XmlRootElement;
2728

29+
import org.apache.ranger.authorization.utils.StringUtil;
2830
import org.codehaus.jackson.annotate.JsonAutoDetect;
2931
import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
3032
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
@@ -152,6 +154,11 @@ public void setVersion(Long version) {
152154
this.version = version;
153155
}
154156

157+
public void dedupStrings(Map<String, String> strTbl) {
158+
createdBy = StringUtil.dedupString(createdBy, strTbl);
159+
updatedBy = StringUtil.dedupString(updatedBy, strTbl);
160+
}
161+
155162
@Override
156163
public String toString( ) {
157164
StringBuilder sb = new StringBuilder();

0 commit comments

Comments
 (0)