[CARBONATA-3605] Remove global dictionary feature

This closes apache#3502

Author: jackylk

conf/carbon.properties.template

@@ -85,7 +85,6 @@ carbon.major.compaction.size=1024
 ##Min max feature is added to enhance query performance. To disable this feature, make it false.
 #carbon.enableMinMax=true
 
-######## Global Dictionary Configurations ########
 ##The property to set the date to be considered as start date for calculating the timestamp.
 #carbon.cutOffTimestamp
 ##The property to set the timestamp (ie milis) conversion to the SECOND, MINUTE, HOUR or DAY level.

conf/dataload.properties.template

@@ -35,10 +35,7 @@ delimiter=,
 #csv comment character
 #commentchar=#
 
-#column dictionary list
-#columndict=
-
-#null value's serialization format  
+#null value's serialization format
 #serialization_null_format=\\N
 
 #bad records logger
@@ -47,9 +44,6 @@ delimiter=,
 #bad records action
 #bad_records_action=force
 
-#all dictionary folder path
-#all_dictionary_path=
-
 #complex column's level 1 delimiter
 #complex_delimiter_level_1='\\\001'
 

core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java

@@ -22,10 +22,6 @@
 import java.util.Map;
 
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
-import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.indexstore.BlockletDataMapIndexStore;
 import org.apache.carbondata.core.util.CarbonProperties;
@@ -89,13 +85,13 @@ public static CacheProvider getInstance() {
   public <K, V> Cache<K, V> createCache(CacheType cacheType) {
     //check if lru cache is null, if null create one
     //check if cache is null for given cache type, if null create one
-    if (!dictionaryCacheAlreadyExists(cacheType)) {
+    if (!isCacheExists(cacheType)) {
       synchronized (lock) {
-        if (!dictionaryCacheAlreadyExists(cacheType)) {
+        if (!isCacheExists(cacheType)) {
           if (null == carbonLRUCache) {
             createLRULevelCacheInstance();
           }
-          createDictionaryCacheForGivenType(cacheType);
+          createBlockletDataMapCache(cacheType);
         }
       }
     }
@@ -110,9 +106,9 @@ public <K, V> Cache<K, V> createCache(CacheType cacheType, String cacheClassName
       throws Exception {
     //check if lru cache is null, if null create one
     //check if cache is null for given cache type, if null create one
-    if (!dictionaryCacheAlreadyExists(cacheType)) {
+    if (!isCacheExists(cacheType)) {
       synchronized (lock) {
-        if (!dictionaryCacheAlreadyExists(cacheType)) {
+        if (!isCacheExists(cacheType)) {
           if (null == carbonLRUCache) {
             createLRULevelCacheInstance();
           }
@@ -132,14 +128,14 @@ public <K, V> Cache<K, V> createCache(CacheType cacheType, String cacheClassName
    *
    * @param cacheType       type of cache
    */
-  private void createDictionaryCacheForGivenType(CacheType cacheType) {
+  private void createBlockletDataMapCache(CacheType cacheType) {
     Cache cacheObject = null;
     if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
-      cacheObject =
-          new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
+      // TODO: remove cache for dictionary after dictionary id deprecated
+      return;
     } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
-      cacheObject =
-          new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
+      // TODO: remove cache for dictionary after dictionary id deprecated
+      return;
     } else if (cacheType.equals(cacheType.DRIVER_BLOCKLET_DATAMAP)) {
       cacheObject = new BlockletDataMapIndexStore(carbonLRUCache);
     }
@@ -181,21 +177,10 @@ private void createLRULevelCacheInstance() {
    * @param cacheType
    * @return
    */
-  private boolean dictionaryCacheAlreadyExists(CacheType cacheType) {
+  private boolean isCacheExists(CacheType cacheType) {
     return null != cacheTypeToCacheMap.get(cacheType);
   }
 
-  /**
-   * Below method will be used to clear the cache
-   */
-  public void dropAllCache() {
-    if (null != carbonLRUCache) {
-      carbonLRUCache.clear();
-      carbonLRUCache = null;
-    }
-    cacheTypeToCacheMap.clear();
-  }
-
   public CarbonLRUCache getCarbonCache() {
     return carbonLRUCache;
   }

core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractColumnDictionaryInfo.java

@@ -17,7 +17,6 @@
 
 package org.apache.carbondata.core.cache.dictionary;
 
-import java.nio.charset.Charset;
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -50,21 +49,11 @@ public abstract class AbstractColumnDictionaryInfo implements DictionaryInfo {
    */
   protected long fileTimeStamp;
 
-  /**
-   * offset till where file is read
-   */
-  protected long offsetTillFileIsRead;
-
   /**
    * memory size of this object.We store it as calculation everytime is costly
    */
   protected long memorySize;
 
-  /**
-   * length of dictionary metadata file
-   */
-  private long dictionaryMetaFileLength;
-
   /**
    * size of one dictionary bucket
    */
@@ -102,31 +91,6 @@ public long getMemorySize() {
     return memorySize;
   }
 
-  @Override
-  public void setMemorySize(long memorySize) {
-    this.memorySize = memorySize;
-  }
-
-  /**
-   * This method will increment the access count for a column by 1
-   * whenever a column is getting used in query or incremental data load
-   */
-  @Override
-  public void incrementAccessCount() {
-    accessCount.incrementAndGet();
-  }
-
-  /**
-   * This method will return the size of of last dictionary chunk so that only that many
-   * values are read from the dictionary reader
-   *
-   * @return size of last dictionary chunk
-   */
-  @Override
-  public int getSizeOfLastDictionaryChunk() {
-    return 0;
-  }
-
   /**
    * This method will decrement the access count for a column by 1
    * whenever a column usage is complete
@@ -137,32 +101,6 @@ private void decrementAccessCount() {
     }
   }
 
-  /**
-   * This method will update the end offset of file everytime a file is read
-   *
-   * @param offsetTillFileIsRead
-   */
-  @Override
-  public void setOffsetTillFileIsRead(long offsetTillFileIsRead) {
-    this.offsetTillFileIsRead = offsetTillFileIsRead;
-  }
-
-  @Override
-  public long getOffsetTillFileIsRead() {
-    return offsetTillFileIsRead;
-  }
-
-  /**
-   * This method will update the timestamp of a file if a file is modified
-   * like in case of incremental load
-   *
-   * @param fileTimeStamp
-   */
-  @Override
-  public void setFileTimeStamp(long fileTimeStamp) {
-    this.fileTimeStamp = fileTimeStamp;
-  }
-
   /**
    * The method return the list of dictionary chunks of a column
    * Applications Scenario.
@@ -183,56 +121,6 @@ public void clear() {
     decrementAccessCount();
   }
 
-  /**
-   * This method will find and return the sort index for a given dictionary id.
-   * Applicable scenarios:
-   * 1. Used in case of order by queries when data sorting is required
-   *
-   * @param surrogateKey a unique ID for a dictionary value
-   * @return if found returns key else 0
-   */
-  @Override
-  public int getSortedIndex(int surrogateKey) {
-    return 0;
-  }
-
-  /**
-   * dictionary metadata file length which will be set whenever we reload dictionary
-   * data from disk
-   *
-   * @param dictionaryMetaFileLength length of dictionary metadata file
-   */
-  @Override
-  public void setDictionaryMetaFileLength(long dictionaryMetaFileLength) {
-    this.dictionaryMetaFileLength = dictionaryMetaFileLength;
-  }
-
-  /**
-   * Dictionary meta file offset which will be read to check whether length of dictionary
-   * meta file has been modified
-   *
-   * @return
-   */
-  @Override
-  public long getDictionaryMetaFileLength() {
-    return dictionaryMetaFileLength;
-  }
-
-  /**
-   * This method will find and return the dictionary value from sorted index.
-   * Applicable scenarios:
-   * 1. Query final result preparation in case of order by queries:
-   * While convert the final result which will
-   * be surrogate key back to original dictionary values this method will be used
-   *
-   * @param sortedIndex sort index of dictionary value
-   * @return value if found else null
-   */
-  @Override
-  public String getDictionaryValueFromSortedIndex(int sortedIndex) {
-    return null;
-  }
-
   /**
    * This method will find and return the dictionary value for a given surrogate key.
    * Applicable scenarios:
@@ -299,22 +187,6 @@ protected byte[] getDictionaryBytesFromSurrogate(int surrogateKey) {
     return dictionaryValueInBytes;
   }
 
-  /**
-   * This method will find and return the surrogate key for a given dictionary value
-   * Applicable scenario:
-   * 1. Incremental data load : Dictionary will not be generated for existing values. For
-   * that values have to be looked up in the existing dictionary cache.
-   * 2. Filter scenarios where from value surrogate key has to be found.
-   *
-   * @param value dictionary value
-   * @return if found returns key else INVALID_SURROGATE_KEY
-   */
-  @Override
-  public int getSurrogateKey(String value) {
-    byte[] keyData = value.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
-    return getSurrogateKey(keyData);
-  }
-
   @Override
   public void invalidate() {