From 54c43fd5b7b08de03e73e7b9ad52b51d0f7180c5 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 17 Mar 2016 17:54:08 +0800 Subject: [PATCH 1/2] Use NavigableSet instead ArrayList to keep rowkey original order instead of urgly using Collections.sort --- .../common/persistence/FileResourceStore.java | 8 ++-- .../common/persistence/ResourceStore.java | 7 +-- .../common/persistence/ResourceTool.java | 7 +-- .../LocalFileResourceStoreTest.java | 3 +- .../apache/kylin/cube/CubeManagerTest.java | 24 ++++------ .../apache/kylin/dict/DictionaryManager.java | 44 ++++++++----------- .../kylin/dict/lookup/SnapshotManager.java | 5 ++- .../apache/kylin/job/dao/ExecutableDao.java | 35 ++++++++------- .../engine/mr/steps/MetadataCleanupJob.java | 15 +++---- .../hbase/ITHBaseResourceStoreTest.java | 3 +- .../apache/kylin/rest/service/JobService.java | 1 + .../storage/hbase/HBaseResourceStore.java | 19 +++----- 12 files changed, 79 insertions(+), 92 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java index 49ff4417bf6..1ab659f0a44 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java @@ -26,6 +26,8 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -49,12 +51,12 @@ public FileResourceStore(KylinConfig kylinConfig) { } @Override - protected ArrayList listResourcesImpl(String resPath) throws IOException { + protected NavigableSet listResourcesImpl(String resPath) throws IOException { String[] names = file(resPath).list(); if (names == null) // not a directory return null; - ArrayList r = new ArrayList(names.length); + TreeSet r = new TreeSet<>(); String prefix = resPath.endsWith("/") ? resPath : resPath + "/"; for (String n : names) { r.add(prefix + n); @@ -75,7 +77,7 @@ protected List getAllResources(String rangeStart, String rangeEnd) try { String commonPrefix = StringUtils.getCommonPrefix(rangeEnd, rangeStart); commonPrefix = commonPrefix.substring(0, commonPrefix.lastIndexOf("/") + 1); - final ArrayList resources = listResourcesImpl(commonPrefix); + final NavigableSet resources = listResourcesImpl(commonPrefix); for (String resource : resources) { if (resource.compareTo(rangeStart) >= 0 && resource.compareTo(rangeEnd) <= 0) { if (existsImpl(resource)) { diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 88ee553a29c..4e9e904c753 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.io.IOUtils; @@ -116,12 +117,12 @@ public ResourceStore(KylinConfig kylinConfig) { * return a list of child resources & folders under given path, return null * if given path is not a folder */ - final public ArrayList listResources(String resPath) throws IOException { + final public NavigableSet listResources(String resPath) throws IOException { resPath = norm(resPath); return listResourcesImpl(resPath); } - abstract protected ArrayList listResourcesImpl(String resPath) throws IOException; + abstract protected NavigableSet listResourcesImpl(String resPath) throws IOException; /** * return true if a resource exists, return false in case of folder or @@ -278,7 +279,7 @@ public static interface Visitor { } public void scanRecursively(String path, Visitor visitor) throws IOException { - ArrayList children = listResources(path); + NavigableSet children = listResources(path); if (children != null) { for (String child : children) scanRecursively(child, visitor); diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java index 2e1d527e9e8..52adeba447d 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java @@ -24,6 +24,7 @@ import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; +import java.util.NavigableSet; import org.apache.commons.io.IOUtils; import org.apache.kylin.common.KylinConfig; @@ -99,7 +100,7 @@ public static void cat(KylinConfig config, String path) throws IOException { public static void list(KylinConfig config, String path) throws IOException { ResourceStore store = ResourceStore.getStore(config); - ArrayList result = store.listResources(path); + NavigableSet result = store.listResources(path); System.out.println("" + result); } @@ -119,7 +120,7 @@ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOE } public static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException { - ArrayList children = src.listResources(path); + NavigableSet children = src.listResources(path); // case of resource (not a folder) if (children == null) { @@ -164,7 +165,7 @@ public static void reset(KylinConfig config) throws IOException { } private static void resetR(ResourceStore store, String path) throws IOException { - ArrayList children = store.listResources(path); + NavigableSet children = store.listResources(path); if (children == null) { // path is a resource (not a folder) if (matchFilter(path)) { store.deleteResource(path); diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java index a62a578936b..7ba532994fc 100644 --- a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableSet; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -91,7 +92,7 @@ void testAStore(ResourceStore store) throws IOException { } // list - ArrayList list; + NavigableSet list; list = store.listResources(dir1); assertTrue(list.contains(path1)); diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 15c6deb0a06..acfea9e737a 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -18,17 +18,6 @@ package org.apache.kylin.cube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.TimeZone; - -import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.JsonUtil; @@ -41,6 +30,11 @@ import org.junit.Before; import org.junit.Test; +import java.util.List; +import java.util.NavigableSet; + +import static org.junit.Assert.*; + /** * @author yangli9 */ @@ -130,12 +124,12 @@ public void testAutoMergeNormal() throws Exception { @Test public void testGetAllCubes() throws Exception { final ResourceStore store = ResourceStore.getStore(getTestConfig()); - final ArrayList cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT); + final NavigableSet cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT); assertTrue(cubePath.size() > 1); - Collections.sort(cubePath); - final List cubes = store.getAllResources(cubePath.get(0), cubePath.get(cubePath.size() - 1), CubeInstance.class, CubeManager.CUBE_SERIALIZER); +// Collections.sort(cubePath); + final List cubes = store.getAllResources(cubePath.first(), cubePath.last(), CubeInstance.class, CubeManager.CUBE_SERIALIZER); assertEquals(cubePath.size(), cubes.size()); - assertEquals(cubePath.size() - 1, store.getAllResources(cubePath.get(1), cubePath.get(cubePath.size() - 1), CubeInstance.class, CubeManager.CUBE_SERIALIZER).size()); + assertEquals(cubePath.size() - 1, store.getAllResources(cubePath.first(), cubePath.last(), CubeInstance.class, CubeManager.CUBE_SERIALIZER).size()); } @Test diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index ce04b5535c1..a719edaa56b 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -18,17 +18,8 @@ package org.apache.kylin.dict; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - +import com.google.common.cache.*; +import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.dimension.Dictionary; @@ -43,12 +34,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import com.google.common.collect.Lists; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.NavigableSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; public class DictionaryManager { @@ -173,7 +167,7 @@ public DictionaryInfo trySaveNewDict(Dictionary newDict, DictionaryInfo newDi private String checkDupByContent(DictionaryInfo dictInfo, Dictionary dict) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList existings = store.listResources(dictInfo.getResourceDir()); + NavigableSet existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; @@ -316,13 +310,13 @@ public TblColRef decideSourceData(DataModelDesc model, boolean usingDict, TblCol private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList existings = store.listResources(dictInfo.getResourceDir()); + NavigableSet existings = store.listResources(dictInfo.getResourceDir()); if (existings == null || existings.isEmpty()) { return null; } - Collections.sort(existings); +// Collections.sort(existings); - final List allResources = MetadataManager.getInstance(config).getStore().getAllResources(existings.get(0), existings.get(existings.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); + final List allResources = MetadataManager.getInstance(config).getStore().getAllResources(existings.first(), existings.last(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); TableSignature input = dictInfo.getInput(); @@ -336,13 +330,13 @@ private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList dictInfos = store.listResources(dictInfo.getResourceDir()); + NavigableSet dictInfos = store.listResources(dictInfo.getResourceDir()); if (dictInfos == null || dictInfos.isEmpty()) { return null; } - Collections.sort(dictInfos); +// Collections.sort(dictInfos); - final List allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.get(0), dictInfos.get(dictInfos.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); + final List allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.first(), dictInfos.last(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); DictionaryInfo largestDict = null; for (DictionaryInfo dictionaryInfo : allResources) { @@ -371,7 +365,7 @@ public void removeDictionaries(String srcTable, String srcCol) throws IOExceptio info.setSourceColumn(srcCol); ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList existings = store.listResources(info.getResourceDir()); + NavigableSet existings = store.listResources(info.getResourceDir()); if (existings == null) return; diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java index 53bf60d9831..eb05a6453a3 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.KylinConfig; @@ -137,7 +138,7 @@ public SnapshotTable trySaveNewSnapshot(SnapshotTable snapshotTable) throws IOEx private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); - ArrayList existings = store.listResources(resourceDir); + NavigableSet existings = store.listResources(resourceDir); if (existings == null) return null; @@ -155,7 +156,7 @@ private String checkDupByInfo(SnapshotTable snapshot) throws IOException { private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); - ArrayList existings = store.listResources(resourceDir); + NavigableSet existings = store.listResources(resourceDir); if (existings == null) return null; diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java index 1869ca7f336..237bce67027 100644 --- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java +++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.KylinConfig; @@ -94,13 +95,13 @@ private long writeJobOutputResource(String path, ExecutableOutputPO output) thro public List getJobOutputs() throws PersistentException { try { - ArrayList resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); + NavigableSet resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); if (resources == null || resources.isEmpty()) { return Collections.emptyList(); } - Collections.sort(resources); - String rangeStart = resources.get(0); - String rangeEnd = resources.get(resources.size() - 1); +// Collections.sort(resources); + String rangeStart = resources.first(); + String rangeEnd = resources.last(); return store.getAllResources(rangeStart, rangeEnd, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -110,13 +111,13 @@ public List getJobOutputs() throws PersistentException { public List getJobOutputs(long timeStartInMillis, long timeEndInMillis) throws PersistentException { try { - ArrayList resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); + NavigableSet resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); if (resources == null || resources.isEmpty()) { return Collections.emptyList(); } - Collections.sort(resources); - String rangeStart = resources.get(0); - String rangeEnd = resources.get(resources.size() - 1); + // Collections.sort(resources); + String rangeStart = resources.first(); + String rangeEnd = resources.last(); return store.getAllResources(rangeStart, rangeEnd, timeStartInMillis, timeEndInMillis, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -126,13 +127,13 @@ public List getJobOutputs(long timeStartInMillis, long timeE public List getJobs() throws PersistentException { try { - final List jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); + final NavigableSet jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); if (jobIds == null || jobIds.isEmpty()) { return Collections.emptyList(); } - Collections.sort(jobIds); - String rangeStart = jobIds.get(0); - String rangeEnd = jobIds.get(jobIds.size() - 1); +// Collections.sort(jobIds); + String rangeStart = jobIds.first(); + String rangeEnd = jobIds.last(); return store.getAllResources(rangeStart, rangeEnd, ExecutablePO.class, JOB_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -142,13 +143,13 @@ public List getJobs() throws PersistentException { public List getJobs(long timeStartInMillis, long timeEndInMillis) throws PersistentException { try { - final List jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); + final NavigableSet jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); if (jobIds == null || jobIds.isEmpty()) { return Collections.emptyList(); } - Collections.sort(jobIds); - String rangeStart = jobIds.get(0); - String rangeEnd = jobIds.get(jobIds.size() - 1); +// Collections.sort(jobIds); + String rangeStart = jobIds.first(); + String rangeEnd = jobIds.last(); return store.getAllResources(rangeStart, rangeEnd, timeStartInMillis, timeEndInMillis, ExecutablePO.class, JOB_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -158,7 +159,7 @@ public List getJobs(long timeStartInMillis, long timeEndInMillis) public List getJobIds() throws PersistentException { try { - ArrayList resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); + NavigableSet resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); if (resources == null) { return Collections.emptyList(); } diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java index 99f8a9feca2..b88a170ed83 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java @@ -18,10 +18,7 @@ package org.apache.kylin.engine.mr.steps; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; +import java.util.*; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; @@ -114,11 +111,11 @@ public void cleanup() throws Exception { // two level resources, snapshot tables and cube statistics for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) { - ArrayList snapshotTables = getStore().listResources(resourceRoot); + NavigableSet snapshotTables = getStore().listResources(resourceRoot); if (snapshotTables != null) { for (String snapshotTable : snapshotTables) { - ArrayList snapshotNames = getStore().listResources(snapshotTable); + NavigableSet snapshotNames = getStore().listResources(snapshotTable); if (snapshotNames != null) for (String snapshot : snapshotNames) { if (!activeResourceList.contains(snapshot)) { @@ -131,13 +128,13 @@ public void cleanup() throws Exception { } // three level resources, only dictionaries - ArrayList dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT); + NavigableSet dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT); for (String table : dictTables) { - ArrayList tableColNames = getStore().listResources(table); + NavigableSet tableColNames = getStore().listResources(table); if (tableColNames != null) for (String tableCol : tableColNames) { - ArrayList dictionaries = getStore().listResources(tableCol); + NavigableSet dictionaries = getStore().listResources(tableCol); if (dictionaries != null) for (String dict : dictionaries) if (!activeResourceList.contains(dict)) { diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java index c560bb850bc..dccdca9897a 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java @@ -27,6 +27,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableSet; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -133,7 +134,7 @@ void testAStore(ResourceStore store) throws IOException { } // list - ArrayList list; + NavigableSet list; list = store.listResources(dir1); assertTrue(list.contains(path1)); diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java index cd8eef92e25..d0c29a4ab42 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -90,6 +90,7 @@ public List listAllJobs(final String cubeName, final String project return jobs.subList(offset, offset + limit); } + public List listAllJobs(final String cubeName, final String projectName, final List statusList, final JobTimeFilterEnum timeFilter) { Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index 6d772401241..b9235ca685a 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -22,11 +22,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; @@ -41,10 +37,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.FilterList; -import org.apache.hadoop.hbase.filter.KeyOnlyFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.*; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; @@ -117,14 +110,14 @@ private String getAllInOneTableName() { } @Override - protected ArrayList listResourcesImpl(String resPath) throws IOException { + protected NavigableSet listResourcesImpl(String resPath) throws IOException { assert resPath.startsWith("/"); String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/"; byte[] startRow = Bytes.toBytes(lookForPrefix); byte[] endRow = Bytes.toBytes(lookForPrefix); endRow[endRow.length - 1]++; - ArrayList result = new ArrayList(); + TreeSet result = new TreeSet<>(); HTableInterface table = getConnection().getTable(getAllInOneTableName()); Scan scan = new Scan(startRow, endRow); @@ -136,8 +129,8 @@ protected ArrayList listResourcesImpl(String resPath) throws IOException assert path.startsWith(lookForPrefix); int cut = path.indexOf('/', lookForPrefix.length()); String child = cut < 0 ? path : path.substring(0, cut); - if (result.contains(child) == false) - result.add(child); +// if (!result.contains(child)) + result.add(child); } } finally { IOUtils.closeQuietly(table); From 3aba052f7b598c6e174742c6eaa2983c84ce2ae7 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 17 Mar 2016 17:57:13 +0800 Subject: [PATCH 2/2] Verify prefix in hbase region level instead of comparing String in client side --- .../org/apache/kylin/storage/hbase/HBaseResourceStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index b9235ca685a..8f5d99b6f3b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -113,6 +113,7 @@ private String getAllInOneTableName() { protected NavigableSet listResourcesImpl(String resPath) throws IOException { assert resPath.startsWith("/"); String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/"; + byte[] prefix = Bytes.toBytes(lookForPrefix); byte[] startRow = Bytes.toBytes(lookForPrefix); byte[] endRow = Bytes.toBytes(lookForPrefix); endRow[endRow.length - 1]++; @@ -121,15 +122,14 @@ protected NavigableSet listResourcesImpl(String resPath) throws IOExcept HTableInterface table = getConnection().getTable(getAllInOneTableName()); Scan scan = new Scan(startRow, endRow); - scan.setFilter(new KeyOnlyFilter()); + scan.setFilter(new PrefixFilter(prefix)); try { ResultScanner scanner = table.getScanner(scan); for (Result r : scanner) { String path = Bytes.toString(r.getRow()); - assert path.startsWith(lookForPrefix); + // assert path.startsWith(lookForPrefix); int cut = path.indexOf('/', lookForPrefix.length()); String child = cut < 0 ? path : path.substring(0, cut); -// if (!result.contains(child)) result.add(child); } } finally {